00000100:00100000:2.0:1787828211.558984:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2da40 x1874670646707200/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787828211.559017:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787828211.559019:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828211.559022:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787828211.559023:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828211.559028:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1787828211.559030:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787828211.559030:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787828211.559033:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787828211.559037:0:40701:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3012 00000100:00080000:2.0:1787828211.559041:0:40701:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787828211.559043:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828211.559045:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787828211.559046:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787828211.559048:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787828211.559049:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000020:00000001:2.0:1787828211.559052:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:2.0:1787828211.559055:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828211.559060:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:2.0:1787828211.559066:0:40701:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787828211.559069:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828211.559071:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787828211.559073:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828211.559075:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828211.559077:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1787828211.559078:0:40701:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1787828211.559080:0:40701:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828211.559081:0:40701:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828211.559084:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787828211.559086:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828211.559088:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787828211.559089:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787828211.559090:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828211.559095:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787828211.559101:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2c000 x1874670646707328/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787828211.559115:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787828211.559117:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828211.559121:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787828211.559127:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828211.559132:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1787828211.559136:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787828211.559141:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787828211.559148:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787828211.559154:0:40701:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3012 00000100:00080000:2.0:1787828211.559158:0:40701:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787828211.559161:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828211.559162:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787828211.559166:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787828211.559170:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787828211.559173:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000020:00000001:2.0:1787828211.559175:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:2.0:1787828211.559177:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828211.559178:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:2.0:1787828211.559182:0:40701:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787828211.559186:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828211.559189:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787828211.559191:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828211.559193:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828211.559195:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1787828211.559197:0:40701:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000008:00000001:2.0:1787828211.559199:0:40701:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828211.559201:0:40701:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828211.559210:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787828211.559213:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828211.559215:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787828211.559218:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787828211.559220:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828211.559227:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787828211.559233:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2f480 x1874670646707456/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787828211.559248:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787828211.559250:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828211.559253:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828212.454772:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828212.454779:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828213.478761:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828213.478768:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828214.502775:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828214.502783:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828215.526749:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828215.526756:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828216.550725:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828216.550734:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828216.678892:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787828216.678899:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1787828216.678905:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1787828216.678913:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787828216.678918:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828216.678927:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787828216.678934:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828216.678940:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787828216.678951:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c70143c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646707584:192.168.203.143@tcp:250:kworker.0 00000100:00000001:2.0:1787828216.678969:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787828216.678979:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1787828216.678988:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828216.679006:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1787828216.679014:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828216.679035:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787828216.679037:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369536448 : -126234340015168 : ffff8d30c70143c0) 00000100:00000001:2.0:1787828216.679041:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1787828216.679090:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:2.0:1787828216.679105:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1787828216.679113:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828216.679118:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828216.679120:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1787828216.679125:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828216.679129:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4297690880 00000100:00000001:2.0:1787828216.679139:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 00000100:00000001:2.0:1787828216.679145:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4297690880 00000100:00000001:2.0:1787828216.679149:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 00000100:00000001:2.0:1787828216.679153:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787828216.679155:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828216.679157:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1787828216.679164:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828216/real 1787828216] req@ffff8d30c70143c0 x1874670646707584/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828241 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1787828216.679174:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787828216.679175:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1787828216.679178:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1787828216.679180:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787828216.679182:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828216.679184:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828216.679186:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787828216.679190:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c70143c0 x1874670646707584/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828241 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787828216.679196:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828216.679199:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787828216.679202:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828216.679203:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828216.679204:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828216.679207:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787828216.679209:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787828216.679214:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828216.679225:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:2.0:1787828216.679234:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1787828216.679241:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c70143c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646707584:192.168.203.143@tcp:250:kworker.0 00000100:00000001:2.0:1787828216.679250:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828216.679253:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828216.679254:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787828216.679258:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1787828216.679262:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787828216.679265:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:2.0:1787828216.679269:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:2.0:1787828216.679277:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787828216.679282:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1787828216.679289:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787828216.679294:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828216.679307:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828216.679322:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1787828216.679325:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828216.679327:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787828216.679328:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1787828216.679332:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1787828216.679334:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787828216.679335:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828216.679339:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787828216.679344:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828216.679348:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787828216.679358:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c7014f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646707712:192.168.203.143@tcp:38:kworker.0 00000100:00000001:2.0:1787828216.679365:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787828216.679370:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1787828216.679372:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828216.679375:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1787828216.679381:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828216.679389:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787828216.679393:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369539328 : -126234340012288 : ffff8d30c7014f00) 00000100:00000001:2.0:1787828216.679406:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1787828216.679423:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:2.0:1787828216.679430:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1787828216.679435:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828216.679439:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828216.679446:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1787828216.679452:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828216.679465:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4297689622 00000100:00000001:2.0:1787828216.679473:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4297690880 00000100:00000001:2.0:1787828216.679483:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4297689622 00000100:00000001:2.0:1787828216.679488:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4297690880 00000100:00000001:2.0:1787828216.679495:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787828216.679505:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828216.679510:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1787828216.679519:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828216/real 1787828216] req@ffff8d30c7014f00 x1874670646707712/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828241 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1787828216.679538:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787828216.679559:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1787828216.679566:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1787828216.679569:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787828216.679579:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828216.679584:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828216.679588:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787828216.679595:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c7014f00 x1874670646707712/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828241 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787828216.679646:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828216.679652:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787828216.679665:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828216.679673:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828216.679680:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828216.679684:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787828216.679689:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787828216.679697:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828216.679716:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:2.0:1787828216.679724:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1787828216.679737:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c7014f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646707712:192.168.203.143@tcp:38:kworker.0 00000100:00000001:2.0:1787828216.679744:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828216.679746:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828216.679748:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787828216.679750:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1787828216.679755:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787828216.679757:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:2.0:1787828216.679758:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:2.0:1787828216.679761:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787828216.679763:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1787828216.679764:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787828216.679765:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828216.679767:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828216.679769:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1787828216.679771:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828216.679773:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787828216.679774:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1787828216.679776:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1787828216.679777:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787828216.679778:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828216.679780:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787828216.679781:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828216.679783:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787828216.679786:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c70170c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646707840:192.168.203.143@tcp:38:kworker.0 00000100:00000001:2.0:1787828216.679789:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787828216.679790:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1787828216.679792:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828216.679793:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1787828216.679795:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828216.679799:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787828216.679800:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369547968 : -126234340003648 : ffff8d30c70170c0) 00000100:00000001:2.0:1787828216.679804:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1787828216.679815:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:2.0:1787828216.679818:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1787828216.679820:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828216.679821:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828216.679823:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1787828216.679824:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828216.679827:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4297689623 00000100:00000001:2.0:1787828216.679829:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4297690880 00000100:00000001:2.0:1787828216.679831:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4297689623 00000100:00000001:2.0:1787828216.679833:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4297690880 00000100:00000001:2.0:1787828216.679835:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787828216.679836:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828216.679837:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1787828216.679840:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828216/real 1787828216] req@ffff8d30c70170c0 x1874670646707840/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828241 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1787828216.679845:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787828216.679846:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1787828216.679848:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1787828216.679849:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787828216.679850:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828216.679851:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828216.679852:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787828216.679855:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c70170c0 x1874670646707840/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828241 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787828216.679859:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828216.679860:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787828216.679862:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828216.679863:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828216.679863:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828216.679864:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787828216.679866:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787828216.679868:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828216.679870:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:2.0:1787828216.679872:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1787828216.679874:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c70170c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646707840:192.168.203.143@tcp:38:kworker.0 00000100:00000001:2.0:1787828216.679877:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1787828216.679878:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787828216.679879:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828216.679880:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787828216.679881:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828216.679882:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787828216.679884:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c7016d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646707968:192.168.203.143@tcp:8:kworker.0 00000100:00000001:2.0:1787828216.679886:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787828216.679887:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1787828216.679888:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828216.679889:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1787828216.679891:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828216.679895:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787828216.679896:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369547008 : -126234340004608 : ffff8d30c7016d00) 00000100:00000001:2.0:1787828216.679898:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1787828216.679904:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:2.0:1787828216.679906:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1787828216.679908:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828216.679909:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828216.679910:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1787828216.679912:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828216.679914:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4297689636 00000100:00000001:2.0:1787828216.679916:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4297690880 00000100:00000001:2.0:1787828216.679917:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4297689636 00000100:00000001:2.0:1787828216.679919:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4297690880 00000100:00000001:2.0:1787828216.679920:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787828216.679922:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828216.679923:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1787828216.679926:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828216/real 1787828216] req@ffff8d30c7016d00 x1874670646707968/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828241 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1787828216.679931:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787828216.679932:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1787828216.679933:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1787828216.679934:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787828216.679935:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828216.679936:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828216.679937:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787828216.679939:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c7016d00 x1874670646707968/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828241 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787828216.679944:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828216.679945:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787828216.679947:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828216.679949:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828216.679950:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828216.679952:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787828216.679953:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787828216.679956:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828216.679960:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:2.0:1787828216.679962:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1787828216.679965:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c7016d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646707968:192.168.203.143@tcp:8:kworker.0 00000100:00000001:2.0:1787828216.679969:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828216.679971:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828216.679973:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787828216.679974:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1787828216.679977:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787828216.679978:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:2.0:1787828216.679980:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:2.0:1787828216.679982:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787828216.679984:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1787828216.679986:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787828216.679988:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828216.679990:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828216.679991:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787828216.679993:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1787828216.679995:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787828216.679996:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:2.0:1787828216.679997:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:2.0:1787828216.679999:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787828216.680000:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1787828216.680002:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787828216.680003:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828216.680004:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828216.680006:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1787828216.680008:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828216.680009:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787828216.680011:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828217.574716:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828217.574724:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828218.598715:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828218.598723:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828219.622692:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828219.622702:0:815:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1787828219.622704:0:815:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1787828219.622707:0:815:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828219.622712:0:815:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1787828219.622714:0:815:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828219.622716:0:815:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828219.622718:0:815:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828220.646704:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828220.646713:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828221.670694:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828221.670702:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828221.798970:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787828221.798976:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1787828221.798980:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1787828221.798983:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787828221.798986:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828221.798991:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787828221.798995:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828221.798999:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787828221.799004:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c7014b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646708096:192.168.203.143@tcp:250:kworker.0 00000100:00000001:2.0:1787828221.799010:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787828221.799015:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1787828221.799018:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828221.799021:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1787828221.799028:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828221.799049:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787828221.799052:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369538368 : -126234340013248 : ffff8d30c7014b40) 00000100:00000001:2.0:1787828221.799058:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1787828221.799106:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:2.0:1787828221.799121:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1787828221.799129:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828221.799133:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828221.799136:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1787828221.799142:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828221.799148:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4297696000 00000100:00000001:2.0:1787828221.799171:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 00000100:00000001:2.0:1787828221.799186:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4297696000 00000100:00000001:2.0:1787828221.799193:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 00000100:00000001:2.0:1787828221.799197:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787828221.799201:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828221.799205:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1787828221.799213:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828221/real 1787828221] req@ffff8d30c7014b40 x1874670646708096/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828246 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1787828221.799227:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787828221.799230:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1787828221.799235:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1787828221.799238:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787828221.799241:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828221.799248:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828221.799253:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787828221.799265:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c7014b40 x1874670646708096/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828246 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787828221.799299:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828221.799306:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787828221.799312:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828221.799317:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828221.799321:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828221.799326:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787828221.799328:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787828221.799332:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828221.799351:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:2.0:1787828221.799355:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1787828221.799361:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c7014b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646708096:192.168.203.143@tcp:250:kworker.0 00000100:00000001:2.0:1787828221.799368:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828221.799371:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828221.799374:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787828221.799377:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1787828221.799382:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787828221.799388:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:2.0:1787828221.799391:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:2.0:1787828221.799397:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787828221.799401:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1787828221.799404:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787828221.799408:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828221.799412:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828221.799432:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1787828221.799436:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828221.799439:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787828221.799443:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1787828221.799445:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1787828221.799446:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787828221.799448:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828221.799454:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787828221.799456:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828221.799460:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787828221.799467:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c70152c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646708224:192.168.203.143@tcp:38:kworker.0 00000100:00000001:2.0:1787828221.799475:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787828221.799477:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1787828221.799480:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828221.799483:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1787828221.799487:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828221.799495:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787828221.799498:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369540288 : -126234340011328 : ffff8d30c70152c0) 00000100:00000001:2.0:1787828221.799502:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1787828221.799528:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:2.0:1787828221.799532:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1787828221.799534:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828221.799536:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828221.799537:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1787828221.799539:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828221.799543:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4297694742 00000100:00000001:2.0:1787828221.799546:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4297696000 00000100:00000001:2.0:1787828221.799559:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4297694742 00000100:00000001:2.0:1787828221.799560:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4297696000 00000100:00000001:2.0:1787828221.799562:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787828221.799563:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828221.799564:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1787828221.799567:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828221/real 1787828221] req@ffff8d30c70152c0 x1874670646708224/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828246 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1787828221.799573:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787828221.799574:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1787828221.799576:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1787828221.799577:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787828221.799578:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828221.799579:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828221.799580:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787828221.799583:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c70152c0 x1874670646708224/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828246 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787828221.799592:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828221.799594:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787828221.799596:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828221.799597:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828221.799597:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828221.799598:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787828221.799600:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787828221.799636:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828221.799638:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:2.0:1787828221.799639:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1787828221.799641:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c70152c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646708224:192.168.203.143@tcp:38:kworker.0 00000100:00000001:2.0:1787828221.799644:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1787828221.799645:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787828221.799646:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828221.799647:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787828221.799649:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828221.799651:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787828221.799653:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c7014000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646708352:192.168.203.143@tcp:38:kworker.0 00000100:00000001:2.0:1787828221.799654:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787828221.799656:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1787828221.799657:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828221.799658:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1787828221.799660:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828221.799664:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787828221.799665:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369535488 : -126234340016128 : ffff8d30c7014000) 00000100:00000001:2.0:1787828221.799668:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1787828221.799675:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:2.0:1787828221.799679:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1787828221.799680:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828221.799681:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828221.799683:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1787828221.799684:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828221.799687:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4297694743 00000100:00000001:2.0:1787828221.799689:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4297696000 00000100:00000001:2.0:1787828221.799691:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4297694743 00000100:00000001:2.0:1787828221.799692:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4297696000 00000100:00000001:2.0:1787828221.799694:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787828221.799695:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828221.799696:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1787828221.799698:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828221/real 1787828221] req@ffff8d30c7014000 x1874670646708352/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828246 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1787828221.799704:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787828221.799705:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1787828221.799707:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1787828221.799709:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787828221.799710:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828221.799712:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828221.799713:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787828221.799717:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c7014000 x1874670646708352/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828246 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787828221.799725:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828221.799727:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787828221.799729:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828221.799731:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828221.799733:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828221.799735:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787828221.799737:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787828221.799739:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828221.799743:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:2.0:1787828221.799746:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1787828221.799750:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c7014000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646708352:192.168.203.143@tcp:38:kworker.0 00000100:00000001:2.0:1787828221.799755:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828221.799757:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828221.799758:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787828221.799760:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1787828221.799762:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787828221.799763:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:2.0:1787828221.799764:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:2.0:1787828221.799766:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787828221.799767:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1787828221.799769:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787828221.799770:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828221.799771:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828221.799772:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787828221.799773:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1787828221.799775:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787828221.799776:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:2.0:1787828221.799778:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:2.0:1787828221.799779:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787828221.799781:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1787828221.799783:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787828221.799784:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828221.799786:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828221.799789:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1787828221.799791:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828221.799793:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787828221.799795:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1787828221.799796:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1787828221.799798:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787828221.799799:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828221.799802:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787828221.799804:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828221.799806:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787828221.799810:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c7017c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646708480:192.168.203.143@tcp:8:kworker.0 00000100:00000001:2.0:1787828221.799814:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787828221.799816:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1787828221.799818:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828221.799820:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1787828221.799823:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828221.799831:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787828221.799832:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369550848 : -126234340000768 : ffff8d30c7017c00) 00000100:00000001:2.0:1787828221.799835:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1787828221.799844:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:2.0:1787828221.799848:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1787828221.799849:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828221.799851:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828221.799852:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1787828221.799854:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828221.799857:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4297694756 00000100:00000001:2.0:1787828221.799858:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4297696000 00000100:00000001:2.0:1787828221.799860:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4297694756 00000100:00000001:2.0:1787828221.799862:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4297696000 00000100:00000001:2.0:1787828221.799863:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787828221.799864:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828221.799866:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1787828221.799868:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828221/real 1787828221] req@ffff8d30c7017c00 x1874670646708480/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828246 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1787828221.799873:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787828221.799874:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1787828221.799876:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1787828221.799877:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787828221.799878:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828221.799879:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828221.799880:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787828221.799882:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c7017c00 x1874670646708480/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828246 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787828221.799887:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828221.799889:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787828221.799890:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828221.799892:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828221.799893:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828221.799894:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787828221.799896:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787828221.799898:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828221.799900:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:2.0:1787828221.799901:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1787828221.799903:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c7017c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646708480:192.168.203.143@tcp:8:kworker.0 00000100:00000001:2.0:1787828221.799906:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828221.799907:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828221.799908:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787828221.799909:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1787828221.799911:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787828221.799911:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:2.0:1787828221.799912:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:2.0:1787828221.799913:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787828221.799914:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1787828221.799916:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787828221.799917:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828221.799918:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828221.799921:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1787828221.799922:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828221.799923:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787828221.799924:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828222.694701:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828222.694709:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828223.718698:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828223.718706:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828224.742694:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828224.742700:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828225.766792:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828225.766801:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828226.790741:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828226.790748:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828226.918854:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787828226.918862:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1787828226.918866:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1787828226.918870:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787828226.918874:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828226.918883:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787828226.918888:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828226.918895:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787828226.918904:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c7017840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646708608:192.168.203.143@tcp:250:kworker.0 00000100:00000001:2.0:1787828226.918913:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787828226.918929:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1787828226.918933:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828226.918937:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1787828226.918951:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828226.918981:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787828226.918985:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369549888 : -126234340001728 : ffff8d30c7017840) 00000100:00000001:2.0:1787828226.918996:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:2.0:1787828226.919055:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828226.919072:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4297701120 00000100:00000001:2.0:1787828226.919087:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 00000100:00000001:2.0:1787828226.919102:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4297701120 00000100:00000001:2.0:1787828226.919114:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 00000100:00000001:2.0:1787828226.919123:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787828226.919126:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828226.919131:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828226.919140:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828226.919156:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1787828226.919158:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:2.0:1787828226.919163:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787828226.919167:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1787828226.919168:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1787828226.919169:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787828226.919171:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828226.919173:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787828226.919177:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828226.919179:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787828226.919183:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c7016940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646708736:192.168.203.143@tcp:38:kworker.0 00000100:00000001:2.0:1787828226.919186:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787828226.919188:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1787828226.919189:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828226.919191:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1787828226.919195:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828226.919203:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787828226.919204:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369546048 : -126234340005568 : ffff8d30c7016940) 00000100:00000001:2.0:1787828226.919210:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:2.0:1787828226.919225:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828226.919230:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4297699862 00000100:00000001:2.0:1787828226.919234:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4297701120 00000100:00000001:2.0:1787828226.919237:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4297699862 00000100:00000001:2.0:1787828226.919239:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4297701120 00000100:00000001:2.0:1787828226.919242:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787828226.919244:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828226.919246:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1787828226.919247:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787828226.919248:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828226.919250:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787828226.919252:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828226.919253:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787828226.919257:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c7017c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646708864:192.168.203.143@tcp:38:kworker.0 00000100:00000001:2.0:1787828226.919259:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787828226.919261:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1787828226.919262:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828226.919264:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1787828226.919266:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828226.919272:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787828226.919273:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369550848 : -126234340000768 : ffff8d30c7017c00) 00000100:00000001:2.0:1787828226.919276:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:2.0:1787828226.919286:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828226.919289:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4297699862 00000100:00000001:2.0:1787828226.919291:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4297701120 00000100:00000001:2.0:1787828226.919293:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4297699862 00000100:00000001:2.0:1787828226.919295:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4297701120 00000100:00000001:2.0:1787828226.919296:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787828226.919298:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828226.919299:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1787828226.919300:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787828226.919301:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828226.919303:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787828226.919304:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828226.919305:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787828226.919307:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c7014000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646708992:192.168.203.143@tcp:8:kworker.0 00000100:00000001:2.0:1787828226.919309:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787828226.919310:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1787828226.919311:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828226.919312:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1787828226.919315:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828226.919319:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787828226.919320:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369535488 : -126234340016128 : ffff8d30c7014000) 00000100:00000001:2.0:1787828226.919323:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:2.0:1787828226.919329:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828226.919331:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4297699875 00000100:00000001:2.0:1787828226.919333:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4297701120 00000100:00000001:2.0:1787828226.919335:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4297699875 00000100:00000001:2.0:1787828226.919337:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4297701120 00000100:00000001:2.0:1787828226.919338:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787828226.919339:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828226.919341:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828226.919343:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828226.919345:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1787828226.919346:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:2.0:1787828226.919347:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787828226.919348:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1787828226.919349:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828226.919351:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828227.814728:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828227.814739:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828228.838745:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828228.838753:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828229.862684:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828229.862693:0:815:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1787828229.862695:0:815:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1787828229.862698:0:815:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828229.862704:0:815:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1787828229.862706:0:815:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828229.862708:0:815:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828229.862710:0:815:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000400:00000100:2.0:1787828229.990999:0:2378:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.143:988 00000400:00000100:2.0:1787828229.991062:0:2378:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.143@tcp at host 192.168.203.143 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1787828229.991085:0:2378:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.143@tcp 00000100:00000001:2.0:1787828229.991105:0:2378:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1787828229.991125:0:2378:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828229.991128:0:2378:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828229.991131:0:2378:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:2.0:1787828229.991143:0:2378:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.143@tcp 00000100:00000001:2.0:1787828229.991149:0:2378:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1787828229.991152:0:2378:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828229.991154:0:2378:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828229.991155:0:2378:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:2.0:1787828229.991161:0:2378:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.143@tcp 00000100:00000001:2.0:1787828229.991167:0:2378:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1787828229.991169:0:2378:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828229.991171:0:2378:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828229.991173:0:2378:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:2.0:1787828229.991178:0:2378:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.143@tcp 00000100:00000001:2.0:1787828229.991184:0:2378:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1787828229.991186:0:2378:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828229.991188:0:2378:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828229.991190:0:2378:0:(events.c:68:request_out_callback()) Process leaving 00010000:00000001:2.0:1787828230.886735:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828230.886744:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828231.910749:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828231.910759:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828232.934680:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828232.934687:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828233.958737:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828233.958746:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828234.982820:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828234.982830:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828236.006801:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828236.006814:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828237.030722:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828237.030730:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828238.054769:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828238.054777:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828239.078785:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828239.078795:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828240.102715:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828240.102724:0:815:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1787828240.102726:0:815:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1787828240.102729:0:815:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828240.102734:0:815:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1787828240.102737:0:815:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828240.102739:0:815:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828240.102741:0:815:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828241.126759:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828241.126767:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828242.150808:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828242.150816:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828243.174782:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828243.174789:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828244.198770:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828244.198779:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828245.222679:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828245.222684:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828246.246741:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828246.246748:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828247.270769:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828247.270777:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828248.294823:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828248.294832:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828249.318790:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828249.318798:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828250.342694:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828250.342700:0:815:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1787828250.342702:0:815:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1787828250.342705:0:815:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828250.342708:0:815:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1787828250.342710:0:815:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828250.342712:0:815:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828250.342713:0:815:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828251.366736:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828251.366743:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828252.390659:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828252.390664:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828253.414741:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828253.414748:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828254.438699:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828254.438707:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828255.462683:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828255.462689:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828256.486686:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828256.486692:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828257.510784:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828257.510793:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787828257.638905:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787828257.638907:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787828257.638911:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000020:00000001:0.0:1787828257.638914:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:0.0:1787828257.638917:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787828257.638919:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:0.0:1787828257.638924:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787828257.638928:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828257.638931:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787828257.638937:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787828257.638941:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828257.638944:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1787828257.638949:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1787828257.638954:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828257.638957:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828257.638963:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787828257.638965:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828257.638968:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787828257.638970:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787828257.638973:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828257.638980:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828257.638987:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d24f00 x1874670646711808/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787828257.638998:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787828257.639001:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828257.639005:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787828257.639009:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828257.639014:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1787828257.639016:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787828257.639018:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787828257.639021:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787828257.639024:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3058 00000100:00080000:0.0:1787828257.639028:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787828257.639031:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787828257.639033:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787828257.639036:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787828257.639038:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787828257.639040:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000020:00000001:0.0:1787828257.639043:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:0.0:1787828257.639047:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787828257.639049:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:0.0:1787828257.639053:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787828257.639057:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828257.639059:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787828257.639063:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787828257.639066:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828257.639069:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1787828257.639072:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1787828257.639075:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828257.639078:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828257.639083:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787828257.639085:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828257.639088:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787828257.639090:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787828257.639092:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828257.639098:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828257.639105:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d24000 x1874670646711936/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787828257.639116:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787828257.639119:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828257.639122:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787828257.639126:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828257.639131:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1787828257.639133:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787828257.639135:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787828257.639139:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787828257.639143:0:468:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3058 00000100:00080000:0.0:1787828257.639148:0:468:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787828257.639151:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787828257.639153:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787828257.639157:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787828257.639159:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787828257.639161:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000020:00000001:0.0:1787828257.639165:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:0.0:1787828257.639168:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787828257.639170:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:0.0:1787828257.639175:0:468:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787828257.639178:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828257.639181:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787828257.639202:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787828257.639206:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828257.639210:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1787828257.639215:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1787828257.639220:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828257.639223:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828257.639230:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787828257.639232:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828257.639236:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787828257.639238:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787828257.639242:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828257.639249:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828257.639258:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d270c0 x1874670646712064/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787828257.639314:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787828257.639317:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828257.639321:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828258.534704:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828258.534711:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828259.558672:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828259.558678:0:815:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1787828259.558680:0:815:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1787828259.558682:0:815:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828259.558686:0:815:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1787828259.558688:0:815:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828259.558689:0:815:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828259.558691:0:815:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828260.582690:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828260.582697:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828261.606698:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828261.606704:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828262.630704:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828262.630709:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828263.654715:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828263.654725:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828264.678699:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828264.678705:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828265.702735:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828265.702743:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828266.726741:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828266.726749:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828267.750852:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828267.750862:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787828267.878884:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828267.878897:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1787828267.878900:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787828267.878913:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787828267.878919:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787828267.878930:0:40701:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 3068 00000100:00080000:2.0:1787828267.878945:0:40701:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787828267.878949:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828267.878952:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787828267.878958:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787828267.878961:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787828267.878965:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:2.0:1787828267.878968:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:2.0:1787828267.878970:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828267.878972:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:2.0:1787828267.878977:0:40701:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787828267.878980:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828267.878984:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787828267.878988:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828267.878992:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828267.878996:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:2.0:1787828267.878999:0:40701:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828267.879016:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787828267.879018:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828267.879022:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787828267.879026:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787828267.879032:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828267.879048:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787828267.879061:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2e580 x1874670646712704/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787828267.879074:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787828267.879079:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828267.879134:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787828267.879139:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828267.879143:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1787828267.879146:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787828267.879148:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787828267.879151:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787828267.879154:0:40701:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3068 00000100:00080000:2.0:1787828267.879159:0:40701:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787828267.879162:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828267.879164:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787828267.879167:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787828267.879169:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787828267.879172:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000020:00000001:2.0:1787828267.879175:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:2.0:1787828267.879177:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828267.879178:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:2.0:1787828267.879182:0:40701:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787828267.879185:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828267.879207:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787828267.879239:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828267.879242:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828267.879244:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1787828267.879251:0:40701:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1787828267.879254:0:40701:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828267.879257:0:40701:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828267.879262:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787828267.879264:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828267.879266:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787828267.879268:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787828267.879271:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828267.879277:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787828267.879302:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2c3c0 x1874670646712832/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787828267.879328:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787828267.879333:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828267.879343:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787828267.879349:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828267.879356:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1787828267.879363:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787828267.879369:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787828267.879375:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787828267.879385:0:40701:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3068 00000100:00080000:2.0:1787828267.879397:0:40701:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787828267.879403:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828267.879407:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787828267.879413:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787828267.879418:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787828267.879424:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000020:00000001:2.0:1787828267.879429:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:2.0:1787828267.879432:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828267.879435:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:2.0:1787828267.879445:0:40701:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787828267.879447:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828267.879456:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787828267.879460:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828267.879464:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828267.879466:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1787828267.879469:0:40701:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1787828267.879473:0:40701:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828267.879476:0:40701:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828267.879487:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787828267.879494:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828267.879499:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787828267.879501:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787828267.879503:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828267.879512:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787828267.879519:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2f0c0 x1874670646712960/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787828267.879529:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787828267.879531:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828267.879535:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787828267.879539:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828267.879543:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1787828267.879545:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787828267.879547:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787828267.879551:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787828267.879561:0:40701:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3068 00000100:00080000:2.0:1787828267.879572:0:40701:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787828267.879575:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828267.879578:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787828267.879581:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787828267.879585:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787828267.879590:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000020:00000001:2.0:1787828267.879595:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:2.0:1787828267.879601:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828267.879608:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:2.0:1787828267.879612:0:40701:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787828267.879692:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828267.879699:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787828267.879708:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828267.879714:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828267.879717:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1787828267.879721:0:40701:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000008:00000001:2.0:1787828267.879725:0:40701:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828267.879731:0:40701:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828267.879756:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787828267.879759:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828267.879762:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787828267.879768:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787828267.879774:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828267.879802:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787828267.879836:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2e940 x1874670646713088/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787828267.879856:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787828267.879864:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828267.879871:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828268.774783:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828268.774793:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828269.798817:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828269.798828:0:815:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1787828269.798832:0:815:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1787828269.798836:0:815:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828269.798842:0:815:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1787828269.798845:0:815:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828269.798848:0:815:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828269.798862:0:815:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828270.822804:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828270.822814:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828271.846781:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828271.846795:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828272.870763:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828272.870770:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828273.894743:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828273.894756:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828274.918724:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828274.918733:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828275.942771:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828275.942784:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828276.966729:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828276.966736:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828277.990718:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828277.990727:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828279.014733:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828279.014740:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828280.038739:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828280.038748:0:815:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1787828280.038751:0:815:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1787828280.038754:0:815:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828280.038760:0:815:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1787828280.038763:0:815:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828280.038765:0:815:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828280.038767:0:815:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828281.062760:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828281.062767:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828282.086705:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828282.086712:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828283.110662:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828283.110669:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787828283.238822:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828283.238838:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1787828283.238841:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787828283.238847:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787828283.238870:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787828283.238880:0:468:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 3083 00000100:00080000:0.0:1787828283.238897:0:468:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787828283.238902:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787828283.238906:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787828283.238912:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787828283.238916:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787828283.238920:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:0.0:1787828283.238924:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:0.0:1787828283.238927:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787828283.238929:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:0.0:1787828283.238936:0:468:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787828283.238940:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828283.238944:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787828283.238949:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787828283.238954:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828283.238958:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1787828283.238962:0:468:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828283.238980:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787828283.238983:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828283.238988:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787828283.238992:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787828283.239000:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828283.239019:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828283.239034:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d252c0 x1874670646714240/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787828283.239051:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787828283.239059:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828283.239113:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787828283.239118:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828283.239122:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1787828283.239125:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787828283.239127:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787828283.239131:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787828283.239135:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3083 00000100:00080000:0.0:1787828283.239141:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787828283.239143:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787828283.239145:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787828283.239148:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787828283.239150:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787828283.239154:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000020:00000001:0.0:1787828283.239184:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:0.0:1787828283.239198:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787828283.239206:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:0.0:1787828283.239224:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787828283.239244:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828283.239251:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787828283.239268:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787828283.239285:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828283.239291:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1787828283.239301:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1787828283.239316:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828283.239323:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828283.239331:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787828283.239334:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828283.239337:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787828283.239339:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787828283.239343:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828283.239354:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828283.239388:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d26940 x1874670646714368/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787828283.239428:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787828283.239433:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828283.239439:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787828283.239462:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828283.239491:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1787828283.239498:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787828283.239504:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787828283.239514:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787828283.239530:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3083 00000100:00080000:0.0:1787828283.239560:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787828283.239567:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787828283.239572:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787828283.239581:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787828283.239588:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787828283.239594:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000020:00000001:0.0:1787828283.239602:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:0.0:1787828283.239609:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787828283.239611:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:0.0:1787828283.239685:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787828283.239698:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828283.239706:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787828283.239717:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787828283.239723:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828283.239729:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1787828283.239738:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1787828283.239745:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828283.239750:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828283.239758:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787828283.239764:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828283.239770:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787828283.239776:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787828283.239784:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828283.239803:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828283.239821:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d27c00 x1874670646714496/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787828283.239880:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787828283.239890:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828283.239917:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787828283.239927:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828283.239948:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1787828283.239956:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787828283.239962:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787828283.239983:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787828283.239993:0:468:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3083 00000100:00080000:0.0:1787828283.240006:0:468:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787828283.240012:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787828283.240019:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787828283.240031:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787828283.240039:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787828283.240046:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000020:00000001:0.0:1787828283.240058:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:0.0:1787828283.240070:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787828283.240076:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:0.0:1787828283.240108:0:468:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787828283.240117:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828283.240126:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787828283.240136:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787828283.240160:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828283.240167:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1787828283.240175:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1787828283.240189:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828283.240196:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828283.240210:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787828283.240215:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828283.240236:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787828283.240246:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787828283.240253:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828283.240274:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828283.240294:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d25680 x1874670646714624/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787828283.240340:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787828283.240350:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828283.240361:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828284.134720:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828284.134727:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828285.158709:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828285.158718:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828286.182690:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828286.182696:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828287.206677:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828287.206682:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828288.230668:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828288.230673:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787828288.358750:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828288.358756:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1787828288.358759:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787828288.358764:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787828288.358769:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787828288.358776:0:468:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 3088 00000100:00080000:0.0:1787828288.358787:0:468:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787828288.358790:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787828288.358793:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787828288.358797:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787828288.358800:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787828288.358804:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:0.0:1787828288.358807:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:0.0:1787828288.358810:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787828288.358812:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:0.0:1787828288.358818:0:468:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787828288.358821:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828288.358824:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787828288.358827:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787828288.358830:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828288.358832:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1787828288.358834:0:468:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828288.358843:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787828288.358845:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828288.358848:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787828288.358850:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787828288.358853:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828288.358863:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828288.358870:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d25680 x1874670646714752/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787828288.358879:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787828288.358882:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828288.358893:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787828288.358896:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828288.358898:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1787828288.358900:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787828288.358902:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787828288.358904:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787828288.358906:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3088 00000100:00080000:0.0:1787828288.358910:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787828288.358912:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787828288.358913:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787828288.358915:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787828288.358916:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787828288.358918:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000020:00000001:0.0:1787828288.358920:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:0.0:1787828288.358922:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787828288.358925:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:0.0:1787828288.358929:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787828288.358933:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828288.358947:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787828288.358953:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787828288.358957:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828288.358958:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1787828288.358962:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1787828288.358967:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828288.358970:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828288.358975:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787828288.358977:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828288.358982:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787828288.358984:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787828288.358987:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828288.358991:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828288.358998:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d27c00 x1874670646714880/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787828288.359004:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787828288.359005:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828288.359007:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787828288.359009:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828288.359011:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1787828288.359012:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787828288.359013:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787828288.359015:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787828288.359017:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3088 00000100:00080000:0.0:1787828288.359018:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787828288.359020:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787828288.359020:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787828288.359022:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787828288.359022:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787828288.359023:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000020:00000001:0.0:1787828288.359024:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:0.0:1787828288.359026:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787828288.359027:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:0.0:1787828288.359029:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787828288.359030:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828288.359032:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787828288.359033:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787828288.359034:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828288.359035:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1787828288.359037:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1787828288.359038:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828288.359039:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828288.359041:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787828288.359042:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828288.359043:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787828288.359044:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787828288.359061:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828288.359066:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828288.359070:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d26940 x1874670646715008/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787828288.359078:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787828288.359081:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828288.359084:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787828288.359086:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828288.359093:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1787828288.359097:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787828288.359099:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787828288.359102:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787828288.359104:0:468:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3088 00000100:00080000:0.0:1787828288.359107:0:468:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787828288.359109:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787828288.359109:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787828288.359111:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787828288.359112:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787828288.359114:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000020:00000001:0.0:1787828288.359115:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:0.0:1787828288.359117:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787828288.359117:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:0.0:1787828288.359120:0:468:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787828288.359121:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828288.359122:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787828288.359124:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787828288.359126:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828288.359127:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1787828288.359128:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1787828288.359130:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828288.359131:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828288.359134:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787828288.359135:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828288.359136:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787828288.359136:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787828288.359138:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828288.359140:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828288.359143:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d252c0 x1874670646715136/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787828288.359148:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787828288.359149:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828288.359150:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828289.254658:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828289.254664:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828290.278670:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828290.278676:0:815:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1787828290.278678:0:815:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1787828290.278680:0:815:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828290.278683:0:815:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1787828290.278685:0:815:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828290.278687:0:815:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828290.278688:0:815:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828291.302694:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828291.302699:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828292.326716:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828292.326722:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828293.350703:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828293.350708:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828294.374754:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828294.374763:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828295.398674:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828295.398680:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828296.422672:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828296.422678:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000400:00000100:2.0:1787828296.550835:0:2379:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.143:988 00000400:00000100:2.0:1787828296.550870:0:2379:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.143@tcp at host 192.168.203.143 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1787828296.550885:0:2379:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.143@tcp 00000100:00000001:2.0:1787828296.550898:0:2379:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1787828296.550912:0:2379:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828296.550914:0:2379:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828296.550915:0:2379:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:2.0:1787828296.550922:0:2379:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.143@tcp 00000100:00000001:2.0:1787828296.550926:0:2379:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1787828296.550928:0:2379:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828296.550930:0:2379:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828296.550931:0:2379:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:2.0:1787828296.550934:0:2379:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.143@tcp 00000100:00000001:2.0:1787828296.550938:0:2379:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1787828296.550940:0:2379:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828296.550941:0:2379:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828296.550942:0:2379:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:2.0:1787828296.550945:0:2379:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.143@tcp 00000100:00000001:2.0:1787828296.550950:0:2379:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1787828296.550952:0:2379:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828296.550953:0:2379:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828296.550956:0:2379:0:(events.c:68:request_out_callback()) Process leaving 00010000:00000001:2.0:1787828297.446698:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828297.446703:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828298.470664:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828298.470669:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828299.494690:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828299.494697:0:815:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1787828299.494698:0:815:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1787828299.494701:0:815:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828299.494704:0:815:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1787828299.494706:0:815:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828299.494708:0:815:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828299.494710:0:815:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828300.518686:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828300.518692:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828301.542729:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828301.542734:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828302.566672:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828302.566677:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828303.590697:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828303.590702:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787828303.718703:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828303.718714:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1787828303.718717:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787828303.718721:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787828303.718725:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787828303.718732:0:40701:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 3104 00000100:00080000:2.0:1787828303.718744:0:40701:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787828303.718748:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828303.718750:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787828303.718755:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787828303.718757:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787828303.718760:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:2.0:1787828303.718763:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:2.0:1787828303.718765:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828303.718767:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:2.0:1787828303.718772:0:40701:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787828303.718788:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828303.718791:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787828303.718795:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828303.718798:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828303.718801:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:2.0:1787828303.718804:0:40701:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828303.718815:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787828303.718818:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828303.718821:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787828303.718823:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787828303.718828:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828303.718839:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787828303.718849:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2c780 x1874670646716288/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787828303.718860:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787828303.718865:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828303.718879:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787828303.718882:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828303.718885:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1787828303.718887:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787828303.718889:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787828303.718893:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787828303.718896:0:40701:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3104 00000100:00080000:2.0:1787828303.718901:0:40701:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787828303.718903:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828303.718906:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787828303.718909:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787828303.718911:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787828303.718917:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000020:00000001:2.0:1787828303.718924:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:2.0:1787828303.718930:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828303.718935:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:2.0:1787828303.718961:0:40701:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787828303.718971:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828303.718976:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787828303.718984:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828303.718991:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828303.718996:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1787828303.719000:0:40701:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1787828303.719004:0:40701:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828303.719010:0:40701:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828303.719019:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787828303.719021:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828303.719023:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787828303.719025:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787828303.719027:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828303.719033:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787828303.719040:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2de00 x1874670646716416/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787828303.719056:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787828303.719060:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828303.719068:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787828303.719076:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828303.719086:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1787828303.719090:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787828303.719092:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787828303.719096:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787828303.719103:0:40701:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3104 00000100:00080000:2.0:1787828303.719108:0:40701:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787828303.719112:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828303.719116:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787828303.719123:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787828303.719128:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787828303.719134:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000020:00000001:2.0:1787828303.719139:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:2.0:1787828303.719147:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828303.719151:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:2.0:1787828303.719158:0:40701:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787828303.719164:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828303.719167:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787828303.719174:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828303.719180:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828303.719185:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1787828303.719189:0:40701:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1787828303.719195:0:40701:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828303.719200:0:40701:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828303.719207:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787828303.719210:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828303.719215:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787828303.719220:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787828303.719225:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828303.719234:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787828303.719248:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2fc00 x1874670646716544/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787828303.719276:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787828303.719281:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828303.719288:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787828303.719293:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828303.719298:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1787828303.719301:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787828303.719306:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787828303.719312:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787828303.719319:0:40701:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3104 00000100:00080000:2.0:1787828303.719327:0:40701:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787828303.719333:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828303.719338:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787828303.719345:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787828303.719349:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787828303.719354:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000020:00000001:2.0:1787828303.719361:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:2.0:1787828303.719368:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828303.719372:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:2.0:1787828303.719379:0:40701:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787828303.719386:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828303.719390:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787828303.719393:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828303.719396:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828303.719401:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1787828303.719407:0:40701:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000008:00000001:2.0:1787828303.719414:0:40701:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828303.719421:0:40701:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828303.719431:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787828303.719436:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828303.719444:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787828303.719449:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787828303.719454:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828303.719464:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787828303.719477:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2d2c0 x1874670646716672/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787828303.719505:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787828303.719510:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828303.719516:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828304.614665:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828304.614671:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828305.638721:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828305.638726:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787828305.638788:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1787828305.638794:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1787828305.638799:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828306.662724:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828306.662744:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828307.686685:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828307.686690:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828308.710704:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828308.710710:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787828308.710773:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1787828308.710778:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1787828308.710781:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787828308.839490:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828308.839492:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787828308.839494:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787828308.839496:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c7016d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646717056:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787828308.839498:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787828308.839499:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787828308.839500:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828308.839501:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787828308.839502:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787828308.839504:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787828308.839505:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c70170c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646717184:192.168.203.143@tcp:8:kworker.0 00000100:00000001:1.0:1787828308.839507:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787828308.839508:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787828308.839509:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828308.839510:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787828308.839512:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828308.839515:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787828308.839516:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369547968 : -126234340003648 : ffff8d30c70170c0) 00000100:00000001:1.0:1787828308.839518:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1787828308.839523:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787828308.839525:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787828308.839527:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787828308.839528:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828308.839529:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787828308.839530:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828308.839532:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4297781795 00000100:00000001:1.0:1787828308.839534:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4297783040 00000100:00000001:1.0:1787828308.839535:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4297781795 00000100:00000001:1.0:1787828308.839537:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4297783040 00000100:00000001:1.0:1787828308.839538:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787828308.839539:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828308.839540:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787828308.839543:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828308/real 1787828308] req@ffff8d30c70170c0 x1874670646717184/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828333 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787828308.839548:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787828308.839549:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787828308.839550:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787828308.839551:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787828308.839552:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828308.839553:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787828308.839554:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787828308.839556:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c70170c0 x1874670646717184/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828333 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787828308.839560:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828308.839561:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787828308.839562:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828308.839563:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787828308.839564:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828308.839565:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787828308.839566:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787828308.839568:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828308.839569:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787828308.839571:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787828308.839572:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c70170c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646717184:192.168.203.143@tcp:8:kworker.0 00000100:00000001:1.0:1787828308.839574:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828308.839576:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787828308.839576:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787828308.839577:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787828308.839579:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787828308.839580:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787828308.839580:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787828308.839582:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787828308.839583:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787828308.839584:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787828308.839585:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828308.839586:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787828308.839587:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787828308.839588:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787828308.839590:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787828308.839590:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787828308.839591:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787828308.839592:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787828308.839593:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787828308.839594:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787828308.839595:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828308.839596:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787828308.839596:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787828308.839597:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787828308.839598:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787828308.839599:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787828308.839620:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787828308.839622:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787828308.839623:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787828308.839624:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787828308.839625:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828308.839626:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828308.839628:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787828308.839629:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828308.839630:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787828308.839632:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828309.734694:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828309.734702:0:815:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1787828309.734704:0:815:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1787828309.734708:0:815:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828309.734713:0:815:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1787828309.734715:0:815:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828309.734717:0:815:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828309.734720:0:815:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787828309.734803:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787828309.734806:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787828309.734809:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828310.758767:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828310.758777:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787828310.758921:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787828310.758927:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787828310.758932:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828311.782741:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828311.782748:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787828311.782862:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1787828311.782869:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1787828311.782874:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828312.806746:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828312.806752:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787828312.806833:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787828312.806836:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787828312.806839:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828313.830682:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828313.830687:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787828313.830743:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787828313.830747:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787828313.830750:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787828313.958791:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828313.958801:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1787828313.958804:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787828313.958808:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787828313.958812:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787828313.958820:0:40701:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 3114 00000100:00080000:2.0:1787828313.958848:0:40701:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787828313.958852:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828313.958854:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787828313.958859:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787828313.958862:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787828313.958865:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:2.0:1787828313.958888:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:2.0:1787828313.958907:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828313.958908:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:2.0:1787828313.958914:0:40701:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787828313.958916:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828313.958919:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787828313.958922:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828313.958924:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828313.958926:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:2.0:1787828313.958929:0:40701:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828313.958940:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787828313.958942:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828313.958944:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787828313.958947:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787828313.958956:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828313.958968:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787828313.958976:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2cb40 x1874670646717312/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787828313.958985:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787828313.958989:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828313.959001:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787828313.959003:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828313.959008:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1787828313.959009:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787828313.959011:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787828313.959013:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787828313.959015:0:40701:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3114 00000100:00080000:2.0:1787828313.959018:0:40701:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787828313.959020:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828313.959021:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787828313.959023:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787828313.959024:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787828313.959026:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000020:00000001:2.0:1787828313.959028:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:2.0:1787828313.959030:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828313.959031:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:2.0:1787828313.959034:0:40701:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787828313.959036:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828313.959037:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:2.0:1787828313.959038:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1787828313.959043:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 02000000:00000001:2.0:1787828313.959045:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787828313.959047:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 02000000:00000001:2.0:1787828313.959049:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828313.959051:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787828313.959055:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828313.959055:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 02000000:00000001:1.0:1787828313.959062:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000008:00100000:2.0:1787828313.959064:0:40701:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 02000000:00000001:1.0:1787828313.959065:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787828313.959070:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:2.0:1787828313.959072:0:40701:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787828313.959076:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646717312:192.168.203.143@tcp:250:kworker.0 00000100:00000001:2.0:1787828313.959078:0:40701:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828313.959086:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787828313.959088:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1787828313.959090:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1787828313.959092:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828313.959094:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828313.959098:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:2.0:1787828313.959098:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787828313.959103:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1787828313.959105:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828313.959108:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828313.959116:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828313.959122:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00100000:2.0:1787828313.959123:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2d680 x1874670646717440/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787828313.959125:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417871168 : -126234291680448 : ffff8d30c9e2cb40) 00000100:00000001:1.0:1787828313.959132:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:2.0:1787828313.959141:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787828313.959144:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828313.959147:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787828313.959150:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828313.959152:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1787828313.959154:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787828313.959156:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787828313.959158:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787828313.959160:0:40701:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3114 00000100:00080000:2.0:1787828313.959164:0:40701:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000800:00000100:1.0:1787828313.959168:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:2.0:1787828313.959169:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828313.959173:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787828313.959176:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787828313.959177:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787828313.959179:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:1.0:1787828313.959182:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000020:00000001:2.0:1787828313.959182:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:2.0:1787828313.959187:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828313.959188:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:1.0:1787828313.959189:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00080000:2.0:1787828313.959193:0:40701:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:1.0:1787828313.959194:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828313.959200:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828313.959201:0:2390:0:(events.c:68:request_out_callback()) Process leaving 02000000:00000001:2.0:1787828313.959205:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1787828313.959207:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828313.959208:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828313.959212:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828313.959213:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4297788160 00000100:00000001:2.0:1787828313.959215:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1787828313.959219:0:40701:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1787828313.959222:0:40701:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828313.959223:0:40701:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828313.959225:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 02000000:00000001:2.0:1787828313.959228:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787828313.959231:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828313.959233:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787828313.959234:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787828313.959236:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828313.959239:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787828313.959242:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2e1c0 x1874670646717568/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787828313.959243:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4297788160 00000100:00000001:1.0:1787828313.959250:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 00000100:00000001:2.0:1787828313.959253:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787828313.959255:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787828313.959256:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828313.959258:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828313.959261:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00000001:2.0:1787828313.959261:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787828313.959266:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1787828313.959271:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828313/real 1787828313] req@ffff8d30c9e2cb40 x1874670646717312/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828338 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828313.959272:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1787828313.959277:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787828313.959280:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787828313.959287:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000400:00000001:1.0:1787828313.959289:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00080000:2.0:1787828313.959293:0:40701:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3114 00000400:00000001:1.0:1787828313.959294:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00080000:2.0:1787828313.959299:0:40701:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1787828313.959300:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787828313.959305:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787828313.959305:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828313.959308:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:1.0:1787828313.959309:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828313.959314:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000020:00000001:2.0:1787828313.959315:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787828313.959319:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:1.0:1787828313.959321:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:2.0:1787828313.959322:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00100000:1.0:1787828313.959329:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2cb40 x1874670646717312/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828338 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:2.0:1787828313.959329:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:2.0:1787828313.959336:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828313.959341:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:1.0:1787828313.959342:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828313.959347:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787828313.959352:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:2.0:1787828313.959352:0:40701:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:1.0:1787828313.959356:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787828313.959359:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828313.959361:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828313.959362:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787828313.959365:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 02000000:00000001:2.0:1787828313.959367:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00080000:1.0:1787828313.959371:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 02000000:00000001:2.0:1787828313.959372:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00080000:1.0:1787828313.959377:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 02000000:00000001:2.0:1787828313.959378:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828313.959382:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000001:2.0:1787828313.959383:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1787828313.959386:0:40701:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000008:00000001:2.0:1787828313.959390:0:40701:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787828313.959393:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646717312:192.168.203.143@tcp:250:kworker.0 00000100:00000001:2.0:1787828313.959394:0:40701:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828313.959402:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:2.0:1787828313.959402:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787828313.959406:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828313.959407:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828313.959411:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:1.0:1787828313.959412:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787828313.959415:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1787828313.959416:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1787828313.959419:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828313.959420:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787828313.959424:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787828313.959425:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 00000100:00000001:2.0:1787828313.959427:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828313.959429:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787828313.959432:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787828313.959436:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787828313.959438:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828313.959441:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787828313.959441:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2d2c0 x1874670646717696/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787828313.959449:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787828313.959452:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828313.959457:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787828313.959459:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787828313.959460:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1787828313.959463:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1787828313.959466:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828313.959467:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787828313.959470:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828313.959473:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787828313.959475:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787828313.959476:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828313.959477:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787828313.959484:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2d680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646717440:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787828313.959487:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787828313.959489:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787828313.959491:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828313.959492:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787828313.959495:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828313.959499:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787828313.959500:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417874048 : -126234291677568 : ffff8d30c9e2d680) 00000100:00000001:1.0:1787828313.959504:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1787828313.959514:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787828313.959518:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787828313.959520:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787828313.959521:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828313.959522:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787828313.959525:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828313.959528:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4297786902 00000100:00000001:1.0:1787828313.959530:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4297788160 00000100:00000001:1.0:1787828313.959532:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4297786902 00000100:00000001:1.0:1787828313.959534:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4297788160 00000100:00000001:1.0:1787828313.959535:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787828313.959537:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828313.959538:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787828313.959541:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828313/real 1787828313] req@ffff8d30c9e2d680 x1874670646717440/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828338 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787828313.959547:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787828313.959548:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787828313.959549:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787828313.959551:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787828313.959552:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828313.959553:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787828313.959555:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787828313.959557:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2d680 x1874670646717440/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828338 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787828313.959563:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828313.959564:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787828313.959566:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828313.959567:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787828313.959568:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828313.959569:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787828313.959570:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787828313.959572:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828313.959574:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787828313.959575:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787828313.959577:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2d680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646717440:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787828313.959580:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787828313.959581:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787828313.959582:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828313.959583:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787828313.959584:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787828313.959586:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787828313.959588:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646717568:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787828313.959590:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787828313.959591:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787828313.959592:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828313.959594:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787828313.959595:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828313.959598:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787828313.959622:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417876928 : -126234291674688 : ffff8d30c9e2e1c0) 00000100:00000001:1.0:1787828313.959625:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1787828313.959634:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787828313.959637:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787828313.959639:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787828313.959640:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828313.959641:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787828313.959643:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828313.959646:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4297786903 00000100:00000001:1.0:1787828313.959648:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4297788160 00000100:00000001:1.0:1787828313.959649:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4297786903 00000100:00000001:1.0:1787828313.959651:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4297788160 00000100:00000001:1.0:1787828313.959653:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787828313.959654:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828313.959655:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787828313.959658:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828313/real 1787828313] req@ffff8d30c9e2e1c0 x1874670646717568/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828338 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787828313.959663:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787828313.959664:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787828313.959666:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787828313.959667:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787828313.959668:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828313.959669:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787828313.959670:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787828313.959673:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2e1c0 x1874670646717568/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828338 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787828313.959677:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828313.959679:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787828313.959680:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828313.959681:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787828313.959682:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828313.959683:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787828313.959684:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787828313.959686:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828313.959688:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787828313.959690:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787828313.959692:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646717568:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787828313.959695:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828313.959696:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787828313.959697:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787828313.959698:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787828313.959700:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787828313.959701:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787828313.959702:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787828313.959703:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787828313.959704:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787828313.959706:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787828313.959707:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828313.959709:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787828313.959710:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787828313.959710:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787828313.959712:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787828313.959713:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787828313.959714:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787828313.959715:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787828313.959716:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787828313.959717:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787828313.959718:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828313.959719:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828313.959721:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787828313.959722:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828313.959724:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787828313.959725:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1787828313.959725:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787828313.959726:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787828313.959727:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828313.959729:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787828313.959730:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787828313.959731:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787828313.959734:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2d2c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646717696:192.168.203.143@tcp:8:kworker.0 00000100:00000001:1.0:1787828313.959736:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787828313.959737:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787828313.959739:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828313.959740:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787828313.959742:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828313.959746:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787828313.959747:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417873088 : -126234291678528 : ffff8d30c9e2d2c0) 00000100:00000001:1.0:1787828313.959749:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1787828313.959756:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787828313.959759:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787828313.959760:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787828313.959761:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828313.959763:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787828313.959764:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828313.959766:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4297786916 00000100:00000001:1.0:1787828313.959768:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4297788160 00000100:00000001:1.0:1787828313.959770:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4297786916 00000100:00000001:1.0:1787828313.959771:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4297788160 00000100:00000001:1.0:1787828313.959773:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787828313.959774:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828313.959775:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787828313.959778:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828313/real 1787828313] req@ffff8d30c9e2d2c0 x1874670646717696/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828338 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787828313.959783:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787828313.959784:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787828313.959786:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787828313.959787:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787828313.959788:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828313.959789:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787828313.959790:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787828313.959792:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2d2c0 x1874670646717696/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828338 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787828313.959797:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828313.959798:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787828313.959800:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828313.959802:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787828313.959803:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828313.959804:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787828313.959805:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787828313.959807:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828313.959809:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787828313.959810:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787828313.959812:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2d2c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646717696:192.168.203.143@tcp:8:kworker.0 00000100:00000001:1.0:1787828313.959814:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828313.959816:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787828313.959816:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787828313.959817:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787828313.959819:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787828313.959819:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787828313.959820:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787828313.959821:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787828313.959822:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787828313.959823:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787828313.959824:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828313.959825:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828313.959827:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787828313.959828:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828313.959829:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787828313.959830:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828314.854702:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828314.854710:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787828314.854793:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1787828314.854798:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1787828314.854802:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828315.878751:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828315.878759:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787828315.878838:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787828315.878845:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787828315.878851:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828316.902867:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828316.902894:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787828316.903002:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787828316.903009:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787828316.903014:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828317.926706:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828317.926714:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787828317.926779:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1787828317.926784:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1787828317.926789:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828318.950869:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828318.950882:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787828318.950986:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787828318.950993:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787828318.950997:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1787828319.079076:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787828319.079084:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1787828319.079089:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787828319.079094:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787828319.079099:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828319.079107:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787828319.079112:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787828319.079117:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787828319.079131:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c7014f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646717824:192.168.203.143@tcp:250:kworker.0 00000100:00000001:1.0:1787828319.079143:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787828319.079150:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787828319.079154:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828319.079161:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787828319.079170:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828319.079193:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787828319.079198:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369539328 : -126234340012288 : ffff8d30c7014f00) 00000100:00000001:1.0:1787828319.079206:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1787828319.079250:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787828319.079264:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787828319.079272:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787828319.079279:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828319.079284:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787828319.079292:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828319.079300:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4297793280 00000100:00000001:1.0:1787828319.079316:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 00000100:00000001:1.0:1787828319.079340:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4297793280 00000100:00000001:1.0:1787828319.079348:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 00000100:00000001:1.0:1787828319.079356:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787828319.079364:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828319.079372:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787828319.079393:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828319/real 1787828319] req@ffff8d30c7014f00 x1874670646717824/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828344 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787828319.079416:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787828319.079421:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787828319.079430:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787828319.079438:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787828319.079444:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828319.079461:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787828319.079465:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787828319.079474:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c7014f00 x1874670646717824/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828344 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787828319.079493:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828319.079498:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787828319.079506:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828319.079511:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787828319.079514:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828319.079519:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787828319.079524:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787828319.079530:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828319.079538:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787828319.079541:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787828319.079545:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c7014f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646717824:192.168.203.143@tcp:250:kworker.0 00000100:00000001:1.0:1787828319.079550:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828319.079553:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787828319.079554:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787828319.079557:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787828319.079560:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787828319.079563:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787828319.079565:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787828319.079569:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787828319.079571:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787828319.079574:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787828319.079576:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828319.079578:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828319.079585:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787828319.079586:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828319.079589:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787828319.079590:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1787828319.079592:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787828319.079593:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787828319.079595:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828319.079597:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787828319.079654:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787828319.079658:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787828319.079664:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c70143c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646717952:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787828319.079669:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787828319.079672:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787828319.079675:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828319.079677:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787828319.079693:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828319.079701:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787828319.079703:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369536448 : -126234340015168 : ffff8d30c70143c0) 00000100:00000001:1.0:1787828319.079709:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1787828319.079727:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787828319.079733:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787828319.079735:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787828319.079737:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828319.079739:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787828319.079742:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828319.079757:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4297792023 00000100:00000001:1.0:1787828319.079760:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4297793280 00000100:00000001:1.0:1787828319.079763:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4297792023 00000100:00000001:1.0:1787828319.079765:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4297793280 00000100:00000001:1.0:1787828319.079767:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787828319.079769:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828319.079771:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787828319.079775:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828319/real 1787828319] req@ffff8d30c70143c0 x1874670646717952/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828344 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787828319.079783:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787828319.079784:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787828319.079786:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787828319.079788:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787828319.079789:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828319.079791:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787828319.079793:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787828319.079796:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c70143c0 x1874670646717952/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828344 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787828319.079803:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828319.079805:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787828319.079808:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828319.079809:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787828319.079810:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828319.079812:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787828319.079813:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787828319.079816:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828319.079818:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787828319.079820:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787828319.079823:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c70143c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646717952:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787828319.079827:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787828319.079828:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787828319.079830:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828319.079853:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787828319.079866:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787828319.079869:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787828319.079872:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c7015680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646718080:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787828319.079875:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787828319.079877:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787828319.079879:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828319.079881:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787828319.079885:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828319.079892:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787828319.079894:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369541248 : -126234340010368 : ffff8d30c7015680) 00000100:00000001:1.0:1787828319.079899:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1787828319.079916:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787828319.079923:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787828319.079934:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787828319.079936:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828319.079939:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787828319.079943:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828319.079949:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4297792023 00000100:00000001:1.0:1787828319.079953:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4297793280 00000100:00000001:1.0:1787828319.079956:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4297792023 00000100:00000001:1.0:1787828319.079959:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4297793280 00000100:00000001:1.0:1787828319.079963:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787828319.079966:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828319.079967:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787828319.079972:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828319/real 1787828319] req@ffff8d30c7015680 x1874670646718080/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828344 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787828319.079981:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787828319.079982:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787828319.079985:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787828319.079986:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787828319.079988:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828319.079989:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787828319.079991:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787828319.079994:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c7015680 x1874670646718080/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828344 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787828319.080000:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828319.080002:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787828319.080010:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828319.080012:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787828319.080013:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828319.080014:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787828319.080016:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787828319.080019:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828319.080022:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787828319.080024:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787828319.080027:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c7015680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646718080:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787828319.080031:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787828319.080032:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787828319.080033:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828319.080035:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787828319.080037:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787828319.080039:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787828319.080041:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c7015a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646718208:192.168.203.143@tcp:8:kworker.0 00000100:00000001:1.0:1787828319.080044:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787828319.080046:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787828319.080048:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828319.080049:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787828319.080052:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828319.080057:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787828319.080059:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369542208 : -126234340009408 : ffff8d30c7015a40) 00000100:00000001:1.0:1787828319.080062:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1787828319.080091:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787828319.080095:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787828319.080098:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787828319.080099:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828319.080101:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787828319.080103:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828319.080107:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4297792036 00000100:00000001:1.0:1787828319.080109:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4297793280 00000100:00000001:1.0:1787828319.080111:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4297792036 00000100:00000001:1.0:1787828319.080113:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4297793280 00000100:00000001:1.0:1787828319.080115:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787828319.080117:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828319.080119:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787828319.080123:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828319/real 1787828319] req@ffff8d30c7015a40 x1874670646718208/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828344 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787828319.080131:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787828319.080132:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787828319.080134:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787828319.080136:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787828319.080137:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828319.080139:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787828319.080140:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787828319.080144:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c7015a40 x1874670646718208/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828344 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787828319.080150:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828319.080151:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787828319.080153:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828319.080155:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787828319.080156:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828319.080157:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787828319.080159:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787828319.080161:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828319.080164:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787828319.080166:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787828319.080169:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c7015a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646718208:192.168.203.143@tcp:8:kworker.0 00000100:00000001:1.0:1787828319.080172:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828319.080174:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787828319.080175:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787828319.080177:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787828319.080179:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787828319.080180:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787828319.080182:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787828319.080184:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787828319.080185:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787828319.080186:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787828319.080188:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828319.080190:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787828319.080191:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787828319.080192:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787828319.080194:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787828319.080195:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787828319.080197:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787828319.080198:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787828319.080199:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787828319.080200:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787828319.080201:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828319.080203:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787828319.080204:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787828319.080205:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787828319.080206:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787828319.080207:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787828319.080208:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787828319.080209:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787828319.080210:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787828319.080211:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787828319.080212:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828319.080214:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828319.080217:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787828319.080219:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828319.080220:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787828319.080222:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828319.974710:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828319.974719:0:815:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1787828319.974721:0:815:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1787828319.974724:0:815:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828319.974729:0:815:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1787828319.974732:0:815:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828319.974733:0:815:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828319.974735:0:815:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787828319.974842:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787828319.974848:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787828319.974852:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828320.998724:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828320.998733:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787828320.998816:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1787828320.998838:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1787828320.998843:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828322.022709:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828322.022719:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787828322.022876:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787828322.022883:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787828322.022888:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828323.046718:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828323.046727:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787828323.046789:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787828323.046795:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787828323.046798:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828324.070768:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828324.070781:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787828324.070886:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1787828324.070892:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1787828324.070896:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1787828324.198852:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787828324.198859:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1787828324.198867:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787828324.198873:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787828324.198879:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828324.198887:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787828324.198894:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787828324.198900:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787828324.198913:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c7016580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646718336:192.168.203.143@tcp:250:kworker.0 00000100:00000001:1.0:1787828324.198923:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787828324.198930:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787828324.198934:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828324.198942:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787828324.198947:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828324.198964:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787828324.198969:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369545088 : -126234340006528 : ffff8d30c7016580) 00000100:00000001:1.0:1787828324.198974:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1787828324.199025:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787828324.199037:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787828324.199042:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787828324.199047:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828324.199052:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787828324.199062:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828324.199068:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4297798400 00000100:00000001:1.0:1787828324.199077:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 00000100:00000001:1.0:1787828324.199084:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4297798400 00000100:00000001:1.0:1787828324.199088:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 00000100:00000001:1.0:1787828324.199094:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787828324.199096:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828324.199098:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787828324.199106:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828324/real 1787828324] req@ffff8d30c7016580 x1874670646718336/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828349 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787828324.199115:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787828324.199117:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787828324.199124:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787828324.199129:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787828324.199136:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828324.199141:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787828324.199144:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787828324.199151:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c7016580 x1874670646718336/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828349 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787828324.199165:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828324.199172:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787828324.199179:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828324.199181:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787828324.199184:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828324.199189:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787828324.199192:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787828324.199199:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828324.199205:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787828324.199211:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787828324.199216:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c7016580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646718336:192.168.203.143@tcp:250:kworker.0 00000100:00000001:1.0:1787828324.199220:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828324.199221:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787828324.199223:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787828324.199225:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787828324.199228:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787828324.199230:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787828324.199231:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787828324.199234:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787828324.199235:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787828324.199238:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787828324.199239:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828324.199240:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828324.199246:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787828324.199247:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828324.199249:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787828324.199250:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1787828324.199251:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787828324.199252:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787828324.199253:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828324.199255:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787828324.199257:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787828324.199258:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787828324.199261:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c70161c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646718464:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787828324.199263:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787828324.199265:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787828324.199266:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828324.199267:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787828324.199269:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828324.199273:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787828324.199274:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369544128 : -126234340007488 : ffff8d30c70161c0) 00000100:00000001:1.0:1787828324.199278:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1787828324.199288:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787828324.199292:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787828324.199293:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787828324.199294:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828324.199295:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787828324.199297:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828324.199300:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4297797142 00000100:00000001:1.0:1787828324.199303:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4297798400 00000100:00000001:1.0:1787828324.199305:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4297797142 00000100:00000001:1.0:1787828324.199306:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4297798400 00000100:00000001:1.0:1787828324.199308:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787828324.199309:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828324.199310:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787828324.199314:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828324/real 1787828324] req@ffff8d30c70161c0 x1874670646718464/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828349 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787828324.199322:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787828324.199325:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787828324.199329:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787828324.199333:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787828324.199337:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828324.199339:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787828324.199340:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787828324.199342:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c70161c0 x1874670646718464/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828349 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787828324.199360:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828324.199362:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787828324.199364:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828324.199366:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787828324.199367:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828324.199375:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787828324.199377:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787828324.199380:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828324.199384:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787828324.199386:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787828324.199395:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c70161c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646718464:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787828324.199400:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787828324.199403:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787828324.199406:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828324.199410:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787828324.199415:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787828324.199422:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787828324.199429:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c7015a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646718592:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787828324.199432:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787828324.199435:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787828324.199438:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828324.199440:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787828324.199443:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828324.199447:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787828324.199449:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369542208 : -126234340009408 : ffff8d30c7015a40) 00000100:00000001:1.0:1787828324.199453:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1787828324.199467:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787828324.199471:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787828324.199473:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787828324.199474:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828324.199475:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787828324.199477:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828324.199479:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4297797142 00000100:00000001:1.0:1787828324.199481:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4297798400 00000100:00000001:1.0:1787828324.199483:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4297797142 00000100:00000001:1.0:1787828324.199484:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4297798400 00000100:00000001:1.0:1787828324.199486:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787828324.199487:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828324.199488:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787828324.199490:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828324/real 1787828324] req@ffff8d30c7015a40 x1874670646718592/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828349 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787828324.199495:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787828324.199496:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787828324.199497:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787828324.199498:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787828324.199499:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828324.199500:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787828324.199501:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787828324.199504:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c7015a40 x1874670646718592/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828349 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787828324.199508:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828324.199509:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787828324.199510:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828324.199511:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787828324.199512:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828324.199513:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787828324.199514:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787828324.199516:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828324.199518:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787828324.199520:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787828324.199522:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c7015a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646718592:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787828324.199525:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828324.199526:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787828324.199527:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787828324.199528:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787828324.199529:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787828324.199530:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787828324.199530:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787828324.199532:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787828324.199533:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787828324.199534:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787828324.199535:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828324.199536:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787828324.199537:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787828324.199538:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787828324.199540:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787828324.199541:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787828324.199541:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787828324.199542:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787828324.199543:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787828324.199544:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787828324.199545:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828324.199546:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828324.199547:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787828324.199548:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828324.199549:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787828324.199550:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1787828324.199551:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787828324.199552:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787828324.199553:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828324.199554:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787828324.199555:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787828324.199557:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787828324.199559:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c7015680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646718720:192.168.203.143@tcp:8:kworker.0 00000100:00000001:1.0:1787828324.199560:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787828324.199562:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787828324.199562:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828324.199563:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787828324.199565:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828324.199568:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787828324.199569:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369541248 : -126234340010368 : ffff8d30c7015680) 00000100:00000001:1.0:1787828324.199572:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1787828324.199578:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787828324.199580:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787828324.199582:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787828324.199583:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828324.199584:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787828324.199585:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828324.199594:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4297797155 00000100:00000001:1.0:1787828324.199595:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4297798400 00000100:00000001:1.0:1787828324.199597:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4297797155 00000100:00000001:1.0:1787828324.199622:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4297798400 00000100:00000001:1.0:1787828324.199624:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787828324.199626:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828324.199627:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787828324.199630:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828324/real 1787828324] req@ffff8d30c7015680 x1874670646718720/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828349 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787828324.199636:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787828324.199637:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787828324.199639:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787828324.199641:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787828324.199641:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828324.199643:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787828324.199644:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787828324.199647:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c7015680 x1874670646718720/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828349 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787828324.199652:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828324.199654:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787828324.199656:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828324.199657:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787828324.199659:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828324.199660:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787828324.199662:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787828324.199665:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828324.199668:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787828324.199670:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787828324.199673:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c7015680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646718720:192.168.203.143@tcp:8:kworker.0 00000100:00000001:1.0:1787828324.199677:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828324.199678:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787828324.199679:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787828324.199680:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787828324.199682:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787828324.199683:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787828324.199685:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787828324.199690:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787828324.199692:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787828324.199694:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787828324.199695:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828324.199697:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828324.199699:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787828324.199700:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828324.199702:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787828324.199703:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828325.094715:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828325.094721:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787828325.094840:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787828325.094845:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787828325.094849:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828326.118735:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828326.118742:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787828326.118825:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787828326.118829:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787828326.118833:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828327.142702:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828327.142709:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787828327.142766:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1787828327.142772:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1787828327.142786:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828328.166746:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828328.166755:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787828328.166839:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787828328.166843:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787828328.166847:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828329.190783:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828329.190796:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787828329.190887:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787828329.190894:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787828329.190898:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1787828329.319122:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787828329.319134:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1787828329.319142:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787828329.319147:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787828329.319154:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828329.319165:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787828329.319173:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787828329.319180:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787828329.319205:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c70143c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646718848:192.168.203.143@tcp:250:kworker.0 00000100:00000001:1.0:1787828329.319217:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787828329.319225:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787828329.319231:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828329.319237:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787828329.319266:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828329.319294:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787828329.319301:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369536448 : -126234340015168 : ffff8d30c70143c0) 00000100:00000001:1.0:1787828329.319314:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1787828329.319416:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787828329.319446:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787828329.319461:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787828329.319469:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828329.319482:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787828329.319497:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828329.319526:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4297803520 00000100:00000001:1.0:1787828329.319542:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 00000100:00000001:1.0:1787828329.319566:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4297803520 00000100:00000001:1.0:1787828329.319590:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 00000100:00000001:1.0:1787828329.319659:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787828329.319669:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828329.319675:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787828329.319688:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828329/real 1787828329] req@ffff8d30c70143c0 x1874670646718848/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828354 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787828329.319707:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787828329.319716:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787828329.319728:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787828329.319739:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787828329.319745:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828329.319753:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787828329.319781:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787828329.319794:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c70143c0 x1874670646718848/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828354 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787828329.319829:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828329.319837:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787828329.319848:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828329.319856:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787828329.319861:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828329.319866:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787828329.319873:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787828329.319879:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828329.319887:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787828329.319890:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787828329.319901:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c70143c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646718848:192.168.203.143@tcp:250:kworker.0 00000100:00000001:1.0:1787828329.319925:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828329.319932:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787828329.319935:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787828329.319939:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787828329.319945:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787828329.319950:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787828329.319955:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787828329.319962:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787828329.319967:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787828329.319973:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787828329.319978:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828329.319984:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828329.320002:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787828329.320007:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828329.320011:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787828329.320014:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1787828329.320016:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787828329.320018:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787828329.320021:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828329.320025:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787828329.320028:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787828329.320031:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787828329.320037:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c7014f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646718976:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787828329.320042:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787828329.320045:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787828329.320047:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828329.320050:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787828329.320054:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828329.320063:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787828329.320064:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369539328 : -126234340012288 : ffff8d30c7014f00) 00000100:00000001:1.0:1787828329.320071:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1787828329.320090:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787828329.320096:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787828329.320099:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787828329.320101:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828329.320103:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787828329.320107:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828329.320112:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4297802263 00000100:00000001:1.0:1787828329.320116:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4297803520 00000100:00000001:1.0:1787828329.320119:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4297802263 00000100:00000001:1.0:1787828329.320122:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4297803520 00000100:00000001:1.0:1787828329.320125:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787828329.320127:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828329.320129:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787828329.320134:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828329/real 1787828329] req@ffff8d30c7014f00 x1874670646718976/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828354 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787828329.320144:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787828329.320146:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787828329.320148:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787828329.320150:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787828329.320152:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828329.320155:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787828329.320157:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787828329.320161:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c7014f00 x1874670646718976/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828354 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787828329.320170:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828329.320173:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787828329.320175:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828329.320177:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787828329.320179:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828329.320180:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787828329.320182:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787828329.320185:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828329.320189:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787828329.320191:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787828329.320195:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c7014f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646718976:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787828329.320199:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787828329.320201:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787828329.320203:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828329.320205:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787828329.320207:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787828329.320210:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787828329.320213:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c70170c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646719104:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787828329.320217:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787828329.320219:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787828329.320220:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828329.320222:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787828329.320226:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828329.320230:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787828329.320232:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369547968 : -126234340003648 : ffff8d30c70170c0) 00000100:00000001:1.0:1787828329.320237:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1787828329.320263:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787828329.320276:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787828329.320279:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787828329.320281:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828329.320284:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787828329.320288:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828329.320293:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4297802263 00000100:00000001:1.0:1787828329.320297:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4297803520 00000100:00000001:1.0:1787828329.320300:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4297802263 00000100:00000001:1.0:1787828329.320304:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4297803520 00000100:00000001:1.0:1787828329.320307:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787828329.320311:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828329.320313:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787828329.320320:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828329/real 1787828329] req@ffff8d30c70170c0 x1874670646719104/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828354 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787828329.320334:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787828329.320337:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787828329.320341:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787828329.320343:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787828329.320345:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828329.320348:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787828329.320367:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787828329.320374:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c70170c0 x1874670646719104/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828354 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787828329.320384:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828329.320387:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787828329.320389:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828329.320391:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787828329.320393:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828329.320395:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787828329.320396:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787828329.320399:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828329.320404:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787828329.320407:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787828329.320411:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c70170c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646719104:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787828329.320416:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787828329.320419:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787828329.320421:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828329.320426:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787828329.320430:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787828329.320433:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787828329.320439:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c7016d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646719232:192.168.203.143@tcp:8:kworker.0 00000100:00000001:1.0:1787828329.320445:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787828329.320448:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787828329.320451:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828329.320454:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787828329.320460:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828329.320471:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787828329.320475:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369547008 : -126234340004608 : ffff8d30c7016d00) 00000100:00000001:1.0:1787828329.320481:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1787828329.320505:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787828329.320528:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787828329.320531:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787828329.320532:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828329.320534:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787828329.320537:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828329.320541:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4297802276 00000100:00000001:1.0:1787828329.320544:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4297803520 00000100:00000001:1.0:1787828329.320546:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4297802276 00000100:00000001:1.0:1787828329.320548:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4297803520 00000100:00000001:1.0:1787828329.320551:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787828329.320553:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828329.320554:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787828329.320559:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828329/real 1787828329] req@ffff8d30c7016d00 x1874670646719232/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828354 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787828329.320570:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787828329.320572:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787828329.320575:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787828329.320578:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787828329.320590:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828329.320593:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787828329.320595:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787828329.320617:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c7016d00 x1874670646719232/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828354 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787828329.320631:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828329.320635:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787828329.320640:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828329.320643:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787828329.320645:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828329.320649:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787828329.320652:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787828329.320657:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828329.320663:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787828329.320667:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787828329.320673:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c7016d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646719232:192.168.203.143@tcp:8:kworker.0 00000100:00000001:1.0:1787828329.320677:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828329.320680:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787828329.320682:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787828329.320684:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787828329.320687:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787828329.320689:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787828329.320691:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787828329.320743:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787828329.320746:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787828329.320749:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787828329.320752:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828329.320754:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787828329.320756:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787828329.320758:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787828329.320800:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787828329.320802:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787828329.320803:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787828329.320805:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787828329.320806:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787828329.320808:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787828329.320809:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828329.320811:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787828329.320812:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787828329.320813:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787828329.320815:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787828329.320816:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787828329.320818:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787828329.320819:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787828329.320820:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787828329.320822:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787828329.320824:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828329.320826:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828329.320830:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787828329.320831:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828329.320833:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787828329.320834:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828330.214720:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828330.214733:0:815:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1787828330.214736:0:815:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1787828330.214741:0:815:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828330.214747:0:815:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1787828330.214750:0:815:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828330.214753:0:815:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828330.214769:0:815:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787828330.214854:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1787828330.214859:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1787828330.214863:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828331.238724:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828331.238732:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787828331.238859:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787828331.238866:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787828331.238871:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828332.262695:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828332.262703:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787828332.262811:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787828332.262817:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787828332.262822:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828333.286792:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828333.286801:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787828333.286894:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1787828333.286900:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1787828333.286905:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828334.310698:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828334.310702:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787828334.310799:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787828334.310804:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787828334.310808:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1787828334.438846:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787828334.438852:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1787828334.438856:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787828334.438861:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787828334.438865:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828334.438873:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787828334.438879:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787828334.438886:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787828334.438897:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c7014b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646719360:192.168.203.143@tcp:250:kworker.0 00000100:00000001:1.0:1787828334.438906:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787828334.438912:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787828334.438916:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828334.438920:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787828334.438927:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828334.438944:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787828334.438947:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369538368 : -126234340013248 : ffff8d30c7014b40) 00000100:00000001:1.0:1787828334.438952:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1787828334.438988:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787828334.439000:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787828334.439005:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787828334.439009:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828334.439014:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787828334.439020:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828334.439026:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4297808640 00000100:00000001:1.0:1787828334.439037:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 00000100:00000001:1.0:1787828334.439047:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4297808640 00000100:00000001:1.0:1787828334.439055:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 00000100:00000001:1.0:1787828334.439060:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787828334.439064:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828334.439068:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787828334.439083:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828334/real 1787828334] req@ffff8d30c7014b40 x1874670646719360/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828359 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787828334.439099:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787828334.439103:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787828334.439107:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787828334.439110:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787828334.439112:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828334.439116:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787828334.439120:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787828334.439132:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c7014b40 x1874670646719360/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828359 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787828334.439141:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828334.439144:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787828334.439146:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828334.439147:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787828334.439148:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828334.439150:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787828334.439151:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787828334.439153:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828334.439158:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787828334.439160:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787828334.439164:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c7014b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646719360:192.168.203.143@tcp:250:kworker.0 00000100:00000001:1.0:1787828334.439169:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828334.439171:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787828334.439173:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787828334.439175:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787828334.439178:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787828334.439180:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787828334.439182:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787828334.439187:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787828334.439189:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787828334.439191:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787828334.439193:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828334.439195:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828334.439202:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787828334.439205:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828334.439207:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787828334.439209:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1787828334.439210:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787828334.439211:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787828334.439212:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828334.439214:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787828334.439215:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787828334.439217:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787828334.439220:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c70152c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646719488:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787828334.439222:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787828334.439224:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787828334.439225:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828334.439226:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787828334.439228:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828334.439233:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787828334.439234:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369540288 : -126234340011328 : ffff8d30c70152c0) 00000100:00000001:1.0:1787828334.439237:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1787828334.439248:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787828334.439251:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787828334.439252:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787828334.439253:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828334.439255:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787828334.439256:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828334.439260:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4297807382 00000100:00000001:1.0:1787828334.439262:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4297808640 00000100:00000001:1.0:1787828334.439263:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4297807382 00000100:00000001:1.0:1787828334.439265:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4297808640 00000100:00000001:1.0:1787828334.439266:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787828334.439268:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828334.439269:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787828334.439272:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828334/real 1787828334] req@ffff8d30c70152c0 x1874670646719488/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828359 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787828334.439277:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787828334.439278:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787828334.439280:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787828334.439281:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787828334.439282:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828334.439283:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787828334.439284:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787828334.439287:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c70152c0 x1874670646719488/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828359 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787828334.439296:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828334.439297:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787828334.439299:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828334.439300:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787828334.439301:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828334.439302:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787828334.439303:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787828334.439305:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828334.439307:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787828334.439308:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787828334.439310:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c70152c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646719488:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787828334.439312:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787828334.439313:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787828334.439314:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828334.439315:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787828334.439316:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787828334.439317:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787828334.439319:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c7017840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646719616:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787828334.439321:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787828334.439322:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787828334.439323:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828334.439324:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787828334.439326:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828334.439329:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787828334.439330:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369549888 : -126234340001728 : ffff8d30c7017840) 00000100:00000001:1.0:1787828334.439332:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1787828334.439338:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787828334.439340:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787828334.439342:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787828334.439343:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828334.439344:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787828334.439346:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828334.439347:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4297807382 00000100:00000001:1.0:1787828334.439349:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4297808640 00000100:00000001:1.0:1787828334.439351:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4297807382 00000100:00000001:1.0:1787828334.439353:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4297808640 00000100:00000001:1.0:1787828334.439355:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787828334.439357:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828334.439358:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787828334.439362:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828334/real 1787828334] req@ffff8d30c7017840 x1874670646719616/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828359 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787828334.439370:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787828334.439372:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787828334.439374:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787828334.439376:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787828334.439377:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828334.439379:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787828334.439380:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787828334.439385:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c7017840 x1874670646719616/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828359 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787828334.439393:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828334.439395:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787828334.439397:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828334.439399:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787828334.439400:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828334.439402:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787828334.439404:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787828334.439407:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828334.439409:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787828334.439411:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787828334.439414:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c7017840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646719616:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787828334.439416:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787828334.439417:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787828334.439418:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828334.439420:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787828334.439421:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787828334.439423:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787828334.439425:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c7014000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646719744:192.168.203.143@tcp:8:kworker.0 00000100:00000001:1.0:1787828334.439427:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787828334.439428:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787828334.439429:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828334.439431:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787828334.439433:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828334.439437:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787828334.439438:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369535488 : -126234340016128 : ffff8d30c7014000) 00000100:00000001:1.0:1787828334.439440:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1787828334.439460:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787828334.439465:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787828334.439468:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787828334.439469:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828334.439471:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787828334.439474:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828334.439478:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4297807395 00000100:00000001:1.0:1787828334.439480:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4297808640 00000100:00000001:1.0:1787828334.439483:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4297807395 00000100:00000001:1.0:1787828334.439485:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4297808640 00000100:00000001:1.0:1787828334.439488:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787828334.439490:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828334.439492:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787828334.439497:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828334/real 1787828334] req@ffff8d30c7014000 x1874670646719744/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828359 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787828334.439507:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787828334.439508:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787828334.439510:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787828334.439511:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787828334.439512:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828334.439513:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787828334.439514:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787828334.439517:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c7014000 x1874670646719744/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828359 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787828334.439521:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828334.439523:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787828334.439525:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828334.439526:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787828334.439526:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828334.439528:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787828334.439529:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787828334.439530:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828334.439532:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787828334.439533:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787828334.439535:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c7014000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646719744:192.168.203.143@tcp:8:kworker.0 00000100:00000001:1.0:1787828334.439537:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828334.439539:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787828334.439539:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787828334.439540:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787828334.439542:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787828334.439543:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787828334.439544:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787828334.439546:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787828334.439547:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787828334.439548:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787828334.439549:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828334.439551:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787828334.439551:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787828334.439552:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787828334.439554:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787828334.439555:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787828334.439556:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787828334.439557:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787828334.439557:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787828334.439558:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787828334.439559:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828334.439560:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787828334.439561:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787828334.439562:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787828334.439562:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787828334.439563:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787828334.439564:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787828334.439565:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787828334.439565:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787828334.439566:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787828334.439567:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828334.439568:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828334.439570:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787828334.439571:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828334.439572:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787828334.439574:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828335.334671:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828335.334677:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787828335.334758:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787828335.334761:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787828335.334763:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828336.358682:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828336.358691:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787828336.358880:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1787828336.358905:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1787828336.358910:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828337.382853:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828337.382864:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787828337.382941:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787828337.382948:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787828337.382953:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828338.406820:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828338.406832:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787828338.406905:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787828338.406912:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787828338.406916:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828339.430682:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828339.430703:0:815:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1787828339.430706:0:815:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1787828339.430708:0:815:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828339.430713:0:815:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1787828339.430715:0:815:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828339.430717:0:815:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828339.430729:0:815:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787828339.430840:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1787828339.430846:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1787828339.430850:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787828339.558761:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828339.558770:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1787828339.558772:0:38:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1787828339.558776:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787828339.558780:0:38:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1787828339.558788:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 3140 00000100:00080000:1.0:1787828339.558798:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1787828339.558800:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787828339.558802:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:1.0:1787828339.558806:0:38:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:1.0:1787828339.558808:0:38:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:1.0:1787828339.558811:0:38:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:1.0:1787828339.558813:0:38:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:1.0:1787828339.558815:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787828339.558816:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:1.0:1787828339.558820:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:1.0:1787828339.558822:0:38:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828339.558824:0:38:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1787828339.558828:0:38:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787828339.558830:0:38:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828339.558832:0:38:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:1.0:1787828339.558834:0:38:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828339.558845:0:38:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1787828339.558863:0:38:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828339.558866:0:38:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1787828339.558868:0:38:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1787828339.558871:0:38:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828339.558881:0:38:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787828339.558888:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d13eed00 x1874670646719872/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787828339.558896:0:38:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787828339.558900:0:38:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828339.558914:0:38:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1787828339.558917:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828339.558919:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1787828339.558920:0:38:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1787828339.558921:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787828339.558924:0:38:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1787828339.558926:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3140 00000100:00080000:1.0:1787828339.558929:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1787828339.558931:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787828339.558933:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:1.0:1787828339.558934:0:38:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:1.0:1787828339.558936:0:38:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:1.0:1787828339.558937:0:38:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000020:00000001:1.0:1787828339.558939:0:38:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:1.0:1787828339.558940:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787828339.558942:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:1.0:1787828339.558947:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:1.0:1787828339.558949:0:38:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828339.558953:0:38:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1787828339.558959:0:38:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787828339.558973:0:38:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828339.558977:0:38:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:1.0:1787828339.558987:0:38:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1787828339.558993:0:38:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828339.558999:0:38:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828339.559005:0:38:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1787828339.559009:0:38:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828339.559014:0:38:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1787828339.559017:0:38:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1787828339.559019:0:38:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828339.559027:0:38:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787828339.559033:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d13ee580 x1874670646720000/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787828339.559046:0:38:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787828339.559049:0:38:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828339.559052:0:38:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1787828339.559054:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828339.559057:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:1.0:1787828339.559058:0:38:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1787828339.559059:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787828339.559061:0:38:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1787828339.559063:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3140 00000100:00080000:1.0:1787828339.559066:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1787828339.559068:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787828339.559068:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:1.0:1787828339.559070:0:38:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:1.0:1787828339.559071:0:38:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:1.0:1787828339.559073:0:38:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000020:00000001:1.0:1787828339.559074:0:38:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:1.0:1787828339.559076:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787828339.559077:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:1.0:1787828339.559079:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:1.0:1787828339.559081:0:38:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828339.559082:0:38:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1787828339.559084:0:38:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787828339.559085:0:38:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828339.559087:0:38:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:1.0:1787828339.559090:0:38:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1787828339.559093:0:38:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828339.559096:0:38:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828339.559100:0:38:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1787828339.559101:0:38:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828339.559102:0:38:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1787828339.559103:0:38:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1787828339.559105:0:38:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828339.559108:0:38:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787828339.559115:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d13ee1c0 x1874670646720128/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787828339.559134:0:38:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787828339.559136:0:38:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828339.559141:0:38:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1787828339.559143:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828339.559145:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1787828339.559146:0:38:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1787828339.559147:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787828339.559149:0:38:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1787828339.559153:0:38:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3140 00000100:00080000:1.0:1787828339.559156:0:38:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1787828339.559157:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787828339.559158:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:1.0:1787828339.559159:0:38:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:1.0:1787828339.559160:0:38:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:1.0:1787828339.559161:0:38:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000020:00000001:1.0:1787828339.559162:0:38:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:1.0:1787828339.559164:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787828339.559167:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:1.0:1787828339.559171:0:38:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:1.0:1787828339.559176:0:38:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828339.559179:0:38:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1787828339.559183:0:38:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787828339.559186:0:38:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828339.559189:0:38:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:1.0:1787828339.559191:0:38:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000008:00000001:1.0:1787828339.559194:0:38:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828339.559198:0:38:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828339.559203:0:38:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1787828339.559206:0:38:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828339.559209:0:38:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1787828339.559210:0:38:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1787828339.559214:0:38:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828339.559219:0:38:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787828339.559227:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d13eda40 x1874670646720256/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787828339.559238:0:38:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787828339.559241:0:38:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828339.559245:0:38:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828340.454753:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828340.454760:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787828340.454833:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787828340.454838:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787828340.454842:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828341.478731:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828341.478740:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787828341.478856:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787828341.478861:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787828341.478865:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828342.502707:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828342.502714:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787828342.502789:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1787828342.502795:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1787828342.502800:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828343.526690:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828343.526715:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787828343.526803:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787828343.526807:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787828343.526811:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828344.550680:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828344.550686:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787828344.550760:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787828344.550764:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787828344.550767:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787828344.678807:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828344.678817:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1787828344.678820:0:38:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787828344.678823:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787828344.678829:0:38:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787828344.678836:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 3145 00000100:00080000:2.0:1787828344.678847:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787828344.678850:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828344.678852:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787828344.678856:0:38:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787828344.678858:0:38:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787828344.678862:0:38:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:2.0:1787828344.678864:0:38:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:2.0:1787828344.678867:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828344.678868:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:2.0:1787828344.678873:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787828344.678876:0:38:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828344.678879:0:38:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787828344.678883:0:38:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828344.678886:0:38:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828344.678888:0:38:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:2.0:1787828344.678891:0:38:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828344.678903:0:38:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787828344.678905:0:38:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828344.678919:0:38:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787828344.678922:0:38:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787828344.678926:0:38:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828344.678940:0:38:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787828344.678953:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2fc00 x1874670646720384/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787828344.678968:0:38:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787828344.678974:0:38:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828344.678994:0:38:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787828344.678999:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828344.679004:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1787828344.679006:0:38:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787828344.679009:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787828344.679013:0:38:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787828344.679017:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3145 00000100:00080000:2.0:1787828344.679038:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787828344.679041:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828344.679043:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787828344.679046:0:38:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787828344.679048:0:38:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787828344.679050:0:38:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000020:00000001:2.0:1787828344.679053:0:38:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:2.0:1787828344.679055:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828344.679057:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:2.0:1787828344.679060:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787828344.679063:0:38:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828344.679065:0:38:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787828344.679071:0:38:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828344.679074:0:38:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828344.679076:0:38:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1787828344.679080:0:38:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1787828344.679085:0:38:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828344.679090:0:38:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828344.679097:0:38:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787828344.679101:0:38:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828344.679103:0:38:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787828344.679105:0:38:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787828344.679107:0:38:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828344.679112:0:38:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787828344.679121:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2de00 x1874670646720512/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787828344.679140:0:38:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787828344.679146:0:38:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828344.679154:0:38:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787828344.679159:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828344.679164:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1787828344.679166:0:38:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787828344.679171:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787828344.679179:0:38:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787828344.679185:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3145 00000100:00080000:2.0:1787828344.679189:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787828344.679193:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828344.679194:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787828344.679196:0:38:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787828344.679198:0:38:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787828344.679200:0:38:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000020:00000001:2.0:1787828344.679212:0:38:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:2.0:1787828344.679220:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828344.679222:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:2.0:1787828344.679227:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787828344.679232:0:38:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828344.679235:0:38:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787828344.679239:0:38:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828344.679242:0:38:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828344.679246:0:38:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1787828344.679249:0:38:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1787828344.679253:0:38:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828344.679257:0:38:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828344.679268:0:38:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787828344.679274:0:38:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828344.679278:0:38:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787828344.679282:0:38:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787828344.679285:0:38:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828344.679294:0:38:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787828344.679305:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2c780 x1874670646720640/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787828344.679315:0:38:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787828344.679319:0:38:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828344.679322:0:38:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787828344.679327:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828344.679332:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1787828344.679334:0:38:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787828344.679336:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787828344.679338:0:38:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787828344.679342:0:38:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3145 00000100:00080000:2.0:1787828344.679350:0:38:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787828344.679352:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828344.679354:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787828344.679359:0:38:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787828344.679363:0:38:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787828344.679365:0:38:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000020:00000001:2.0:1787828344.679369:0:38:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:2.0:1787828344.679373:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828344.679377:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:2.0:1787828344.679384:0:38:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787828344.679390:0:38:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828344.679394:0:38:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787828344.679398:0:38:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828344.679405:0:38:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828344.679410:0:38:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1787828344.679414:0:38:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000008:00000001:2.0:1787828344.679422:0:38:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828344.679426:0:38:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828344.679434:0:38:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787828344.679438:0:38:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828344.679443:0:38:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787828344.679445:0:38:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787828344.679447:0:38:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828344.679450:0:38:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787828344.679457:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2e940 x1874670646720768/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787828344.679478:0:38:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787828344.679483:0:38:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828344.679486:0:38:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828345.574742:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828345.574751:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787828345.574840:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1787828345.574846:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1787828345.574851:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828346.598779:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828346.598786:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787828346.598884:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787828346.598892:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787828346.598899:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828347.622693:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828347.622698:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787828347.622755:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787828347.622758:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787828347.622761:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828348.646717:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828348.646724:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787828348.646772:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1787828348.646775:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1787828348.646779:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828349.670744:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828349.670754:0:815:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1787828349.670757:0:815:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1787828349.670761:0:815:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828349.670768:0:815:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1787828349.670771:0:815:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828349.670773:0:815:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828349.670776:0:815:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787828349.670873:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787828349.670881:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787828349.670888:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787828349.798734:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828349.798746:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1787828349.798750:0:38:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787828349.798755:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787828349.798761:0:38:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787828349.798770:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 3150 00000100:00080000:2.0:1787828349.798782:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787828349.798786:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828349.798789:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787828349.798794:0:38:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787828349.798798:0:38:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787828349.798802:0:38:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:2.0:1787828349.798805:0:38:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:2.0:1787828349.798808:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828349.798810:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:2.0:1787828349.798816:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787828349.798820:0:38:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828349.798836:0:38:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787828349.798840:0:38:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828349.798844:0:38:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828349.798847:0:38:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:2.0:1787828349.798851:0:38:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828349.798865:0:38:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787828349.798867:0:38:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828349.798872:0:38:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787828349.798875:0:38:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787828349.798881:0:38:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828349.798895:0:38:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787828349.798907:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2f0c0 x1874670646720896/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787828349.798922:0:38:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787828349.798927:0:38:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828349.798948:0:38:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787828349.798952:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828349.798956:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1787828349.798959:0:38:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787828349.798972:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787828349.798975:0:38:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787828349.798977:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3150 00000100:00080000:2.0:1787828349.798982:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787828349.798984:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828349.798986:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787828349.798989:0:38:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787828349.798991:0:38:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787828349.798994:0:38:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000020:00000001:2.0:1787828349.798998:0:38:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:2.0:1787828349.799001:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828349.799003:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:2.0:1787828349.799008:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787828349.799011:0:38:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828349.799014:0:38:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787828349.799030:0:38:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828349.799037:0:38:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828349.799044:0:38:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1787828349.799057:0:38:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1787828349.799072:0:38:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828349.799081:0:38:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828349.799098:0:38:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787828349.799106:0:38:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828349.799117:0:38:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787828349.799125:0:38:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787828349.799135:0:38:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828349.799154:0:38:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787828349.799175:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2c3c0 x1874670646721024/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787828349.799202:0:38:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787828349.799208:0:38:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828349.799215:0:38:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787828349.799222:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828349.799229:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1787828349.799233:0:38:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787828349.799238:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787828349.799242:0:38:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787828349.799247:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3150 00000100:00080000:2.0:1787828349.799258:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787828349.799266:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828349.799269:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787828349.799273:0:38:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787828349.799286:0:38:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787828349.799289:0:38:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000020:00000001:2.0:1787828349.799293:0:38:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:2.0:1787828349.799297:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828349.799305:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:2.0:1787828349.799312:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787828349.799320:0:38:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828349.799332:0:38:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787828349.799340:0:38:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828349.799346:0:38:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828349.799350:0:38:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1787828349.799358:0:38:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1787828349.799368:0:38:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828349.799376:0:38:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828349.799387:0:38:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787828349.799394:0:38:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828349.799398:0:38:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787828349.799401:0:38:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787828349.799406:0:38:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828349.799424:0:38:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787828349.799447:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2e580 x1874670646721152/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787828349.799482:0:38:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787828349.799489:0:38:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828349.799500:0:38:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787828349.799505:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828349.799513:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1787828349.799520:0:38:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787828349.799523:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787828349.799537:0:38:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787828349.799550:0:38:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3150 00000100:00080000:2.0:1787828349.799562:0:38:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787828349.799568:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828349.799577:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787828349.799588:0:38:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787828349.799591:0:38:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787828349.799595:0:38:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000020:00000001:2.0:1787828349.799670:0:38:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:2.0:1787828349.799674:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828349.799677:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:2.0:1787828349.799683:0:38:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787828349.799687:0:38:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828349.799689:0:38:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787828349.799695:0:38:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828349.799698:0:38:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828349.799703:0:38:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1787828349.799720:0:38:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000008:00000001:2.0:1787828349.799731:0:38:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828349.799739:0:38:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828349.799747:0:38:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787828349.799753:0:38:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828349.799756:0:38:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787828349.799758:0:38:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787828349.799761:0:38:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828349.799775:0:38:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787828349.799792:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2f480 x1874670646721280/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787828349.799824:0:38:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787828349.799833:0:38:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828349.799844:0:38:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828350.694731:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828350.694739:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787828350.694840:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787828350.694846:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787828350.694851:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828351.718771:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828351.718780:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787828351.718886:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1787828351.718892:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1787828351.718897:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828352.742708:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828352.742714:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787828352.742781:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787828352.742785:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787828352.742789:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828353.766719:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828353.766726:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787828353.766833:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787828353.766840:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787828353.766846:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828354.790715:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828354.790726:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787828354.790804:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1787828354.790809:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1787828354.790814:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:1.0:1787828354.918967:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787828354.918974:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1787828354.918979:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787828354.918983:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787828354.918988:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828354.918996:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787828354.919001:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787828354.919008:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787828354.919030:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c7016940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646721408:192.168.203.143@tcp:250:kworker.0 00000100:00000001:1.0:1787828354.919042:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787828354.919048:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787828354.919052:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828354.919061:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787828354.919069:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828354.919101:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787828354.919106:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369546048 : -126234340005568 : ffff8d30c7016940) 00000100:00000001:1.0:1787828354.919115:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1787828354.919175:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787828354.919193:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787828354.919204:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787828354.919211:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828354.919220:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787828354.919231:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828354.919242:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4297829120 00000100:00000001:1.0:1787828354.919264:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 00000100:00000001:1.0:1787828354.919285:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4297829120 00000100:00000001:1.0:1787828354.919294:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 00000100:00000001:1.0:1787828354.919298:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787828354.919307:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828354.919314:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787828354.919329:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828354/real 1787828354] req@ffff8d30c7016940 x1874670646721408/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828379 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787828354.919382:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787828354.919386:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787828354.919394:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787828354.919400:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787828354.919403:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828354.919406:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787828354.919408:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787828354.919415:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c7016940 x1874670646721408/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828379 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787828354.919424:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828354.919427:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787828354.919430:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828354.919431:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787828354.919432:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828354.919434:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787828354.919436:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787828354.919450:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828354.919453:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787828354.919455:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787828354.919459:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c7016940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646721408:192.168.203.143@tcp:250:kworker.0 00000100:00000001:1.0:1787828354.919463:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828354.919465:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787828354.919467:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787828354.919469:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787828354.919472:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787828354.919474:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787828354.919476:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787828354.919480:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787828354.919482:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787828354.919484:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787828354.919486:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828354.919488:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828354.919495:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787828354.919497:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828354.919499:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787828354.919501:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1787828354.919502:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787828354.919504:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787828354.919505:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828354.919507:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787828354.919509:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787828354.919511:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787828354.919514:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c7017c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646721536:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787828354.919517:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787828354.919519:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787828354.919521:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828354.919523:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787828354.919526:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828354.919531:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787828354.919533:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369550848 : -126234340000768 : ffff8d30c7017c00) 00000100:00000001:1.0:1787828354.919537:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1787828354.919551:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787828354.919555:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787828354.919557:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787828354.919559:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828354.919560:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787828354.919563:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828354.919567:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4297827862 00000100:00000001:1.0:1787828354.919570:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4297829120 00000100:00000001:1.0:1787828354.919572:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4297827862 00000100:00000001:1.0:1787828354.919574:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4297829120 00000100:00000001:1.0:1787828354.919576:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787828354.919578:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828354.919579:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787828354.919631:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828354/real 1787828354] req@ffff8d30c7017c00 x1874670646721536/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828379 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787828354.919647:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787828354.919649:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787828354.919652:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787828354.919654:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787828354.919655:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828354.919657:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787828354.919658:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787828354.919663:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c7017c00 x1874670646721536/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828379 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787828354.919669:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828354.919672:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787828354.919674:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828354.919675:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787828354.919676:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828354.919678:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787828354.919679:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787828354.919682:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828354.919684:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787828354.919686:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787828354.919689:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c7017c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646721536:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787828354.919692:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787828354.919693:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787828354.919695:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828354.919697:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787828354.919699:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787828354.919710:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787828354.919713:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c7014000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646721664:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787828354.919716:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787828354.919717:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787828354.919720:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828354.919721:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787828354.919725:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828354.919729:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787828354.919730:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369535488 : -126234340016128 : ffff8d30c7014000) 00000100:00000001:1.0:1787828354.919734:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1787828354.919746:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787828354.919750:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787828354.919752:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787828354.919753:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828354.919755:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787828354.919757:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828354.919760:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4297827863 00000100:00000001:1.0:1787828354.919763:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4297829120 00000100:00000001:1.0:1787828354.919765:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4297827863 00000100:00000001:1.0:1787828354.919767:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4297829120 00000100:00000001:1.0:1787828354.919769:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787828354.919771:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828354.919772:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787828354.919776:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828354/real 1787828354] req@ffff8d30c7014000 x1874670646721664/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828379 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787828354.919782:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787828354.919783:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787828354.919785:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787828354.919786:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787828354.919787:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828354.919789:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787828354.919790:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787828354.919794:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c7014000 x1874670646721664/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828379 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787828354.919799:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828354.919801:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787828354.919804:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828354.919805:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787828354.919806:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828354.919808:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787828354.919809:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787828354.919812:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828354.919814:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787828354.919816:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787828354.919818:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c7014000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646721664:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787828354.919821:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787828354.919822:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787828354.919823:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828354.919825:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787828354.919827:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787828354.919828:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787828354.919831:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c7017840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646721792:192.168.203.143@tcp:8:kworker.0 00000100:00000001:1.0:1787828354.919833:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787828354.919834:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787828354.919836:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828354.919837:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787828354.919839:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828354.919843:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787828354.919844:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369549888 : -126234340001728 : ffff8d30c7017840) 00000100:00000001:1.0:1787828354.919847:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1787828354.919854:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787828354.919857:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787828354.919859:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787828354.919861:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828354.919862:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787828354.919864:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828354.919867:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4297827876 00000100:00000001:1.0:1787828354.919869:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4297829120 00000100:00000001:1.0:1787828354.919872:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4297827876 00000100:00000001:1.0:1787828354.919874:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4297829120 00000100:00000001:1.0:1787828354.919876:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787828354.919878:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828354.919879:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787828354.919882:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828354/real 1787828354] req@ffff8d30c7017840 x1874670646721792/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828379 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787828354.919889:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787828354.919890:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787828354.919891:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787828354.919892:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787828354.919893:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828354.919895:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787828354.919896:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787828354.919899:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c7017840 x1874670646721792/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828379 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787828354.919904:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828354.919906:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787828354.919908:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828354.919910:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787828354.919911:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828354.919913:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787828354.919914:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787828354.919916:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828354.919919:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787828354.919920:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787828354.919923:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c7017840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646721792:192.168.203.143@tcp:8:kworker.0 00000100:00000001:1.0:1787828354.919926:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828354.919927:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787828354.919929:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787828354.919931:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787828354.919933:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787828354.919934:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787828354.919935:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787828354.919938:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787828354.919939:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787828354.919941:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787828354.919942:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828354.919943:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787828354.919945:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787828354.919946:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787828354.919948:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787828354.919949:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787828354.919950:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787828354.919951:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787828354.919952:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787828354.919953:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787828354.919954:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828354.919956:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787828354.919957:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787828354.919957:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787828354.919969:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787828354.919969:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787828354.919970:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787828354.919972:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787828354.919972:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787828354.919974:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787828354.919975:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828354.919976:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828354.919979:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787828354.919980:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828354.919982:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787828354.919983:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828355.814700:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828355.814707:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787828355.814777:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787828355.814781:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787828355.814785:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828356.838711:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828356.838718:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787828356.838801:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787828356.838805:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787828356.838809:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828357.862725:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828357.862782:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787828357.862866:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1787828357.862872:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1787828357.862877:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828358.886709:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828358.886716:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787828358.886869:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787828358.886877:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787828358.886885:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828359.910676:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828359.910683:0:815:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1787828359.910685:0:815:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1787828359.910687:0:815:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828359.910691:0:815:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1787828359.910693:0:815:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828359.910694:0:815:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828359.910696:0:815:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787828359.910773:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787828359.910778:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787828359.910782:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1787828360.038931:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787828360.038937:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787828360.038943:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1787828360.038949:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787828360.038953:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828360.038962:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787828360.038968:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787828360.038975:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828360.038983:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c70152c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646721920:192.168.203.143@tcp:250:kworker.0 00000100:00000001:0.0:1787828360.038989:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787828360.038993:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787828360.038995:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828360.038997:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1787828360.039000:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828360.039012:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787828360.039013:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369540288 : -126234340011328 : ffff8d30c70152c0) 00000100:00000001:0.0:1787828360.039016:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:0.0:1787828360.039055:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828360.039060:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4297834240 00000100:00000001:0.0:1787828360.039068:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1787828360.039078:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4297834240 00000100:00000001:0.0:1787828360.039085:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1787828360.039089:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787828360.039092:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828360.039097:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828360.039103:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828360.039111:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787828360.039115:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:0.0:1787828360.039121:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787828360.039125:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787828360.039129:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828360.039134:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828360.039188:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787828360.039191:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787828360.039192:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1787828360.039193:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787828360.039194:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828360.039196:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787828360.039198:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787828360.039200:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828360.039204:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c7014b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646722048:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787828360.039208:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787828360.039211:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787828360.039213:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828360.039214:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1787828360.039217:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828360.039224:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787828360.039225:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369538368 : -126234340013248 : ffff8d30c7014b40) 00000100:00000001:0.0:1787828360.039228:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:0.0:1787828360.039240:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828360.039244:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4297832982 00000100:00000001:0.0:1787828360.039247:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4297834240 00000100:00000001:0.0:1787828360.039249:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4297832982 00000100:00000001:0.0:1787828360.039251:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4297834240 00000100:00000001:0.0:1787828360.039253:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787828360.039256:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828360.039259:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828360.039261:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828360.039264:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787828360.039265:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:0.0:1787828360.039266:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787828360.039267:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787828360.039269:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828360.039270:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828360.039460:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787828360.039472:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1787828360.039478:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787828360.039484:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787828360.039496:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828360.039509:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787828360.039516:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787828360.039525:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787828360.039540:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c7016d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646722176:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787828360.039564:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787828360.039572:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787828360.039578:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828360.039584:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787828360.039595:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828360.039669:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787828360.039672:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369547008 : -126234340004608 : ffff8d30c7016d00) 00000100:00000001:1.0:1787828360.039679:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:1.0:1787828360.039724:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828360.039734:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4297832983 00000100:00000001:1.0:1787828360.039740:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4297834240 00000100:00000001:1.0:1787828360.039743:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4297832983 00000100:00000001:1.0:1787828360.039746:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4297834240 00000100:00000001:1.0:1787828360.039750:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787828360.039753:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828360.039763:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828360.039766:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828360.039775:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787828360.039778:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:1.0:1787828360.039782:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787828360.039784:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1787828360.039787:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787828360.039790:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787828360.039792:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828360.039797:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787828360.039801:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787828360.039804:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787828360.039811:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c70170c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646722304:192.168.203.143@tcp:8:kworker.0 00000100:00000001:1.0:1787828360.039816:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787828360.039820:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787828360.039822:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828360.039825:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787828360.039829:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828360.039837:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787828360.039838:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369547968 : -126234340003648 : ffff8d30c70170c0) 00000100:00000001:1.0:1787828360.039843:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:1.0:1787828360.039856:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828360.039861:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4297832996 00000100:00000001:1.0:1787828360.039864:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4297834240 00000100:00000001:1.0:1787828360.039867:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4297832996 00000100:00000001:1.0:1787828360.039869:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4297834240 00000100:00000001:1.0:1787828360.039871:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787828360.039874:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828360.039877:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828360.039879:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828360.039882:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787828360.039884:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:1.0:1787828360.039886:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787828360.039887:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1787828360.039889:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828360.039891:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828360.934687:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828360.934696:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787828360.934757:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1787828360.934762:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1787828360.934765:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828361.958688:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828361.958695:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787828361.958758:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787828361.958764:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787828361.958768:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828362.982805:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828362.982815:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787828362.982957:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787828362.982964:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787828362.982970:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:1.0:1787828363.111058:0:2380:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.143:988 00000400:00000100:1.0:1787828363.111135:0:2380:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.143@tcp at host 192.168.203.143 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1787828363.111163:0:2380:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.143@tcp 00000100:00000001:1.0:1787828363.111199:0:2380:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787828363.111228:0:2380:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787828363.111232:0:2380:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828363.111236:0:2380:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:1.0:1787828363.111250:0:2380:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.143@tcp 00000100:00000001:1.0:1787828363.111258:0:2380:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787828363.111263:0:2380:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787828363.111266:0:2380:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828363.111269:0:2380:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:1.0:1787828363.111276:0:2380:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.143@tcp 00000100:00000001:1.0:1787828363.111286:0:2380:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787828363.111291:0:2380:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787828363.111293:0:2380:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828363.111295:0:2380:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:1.0:1787828363.111302:0:2380:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.143@tcp 00000100:00000001:1.0:1787828363.111312:0:2380:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787828363.111315:0:2380:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787828363.111317:0:2380:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828363.111319:0:2380:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787828363.111347:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787828363.111377:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787828363.111382:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787828363.111401:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828360/real 1787828363] req@ffff8d30c70170c0 x1874670646722304/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828385 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787828363.111423:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787828363.111427:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787828363.111440:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787828363.111444:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787828363.111447:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828363.111451:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828363.111453:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787828363.111460:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c70170c0 x1874670646722304/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828385 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787828363.111469:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828363.111474:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787828363.111478:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828363.111481:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828363.111482:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828363.111486:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787828363.111489:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787828363.111494:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828363.111499:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787828363.111503:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787828363.111508:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c70170c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646722304:192.168.203.143@tcp:8:kworker.0 00000100:00000001:0.0:1787828363.111514:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787828363.111521:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828360/real 1787828363] req@ffff8d30c7016d00 x1874670646722176/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828385 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787828363.111553:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787828363.111555:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787828363.111560:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787828363.111562:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787828363.111563:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828363.111566:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828363.111567:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787828363.111572:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c7016d00 x1874670646722176/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828385 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787828363.111581:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828363.111584:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787828363.111587:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828363.111589:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828363.111590:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828363.111593:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787828363.111595:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787828363.111598:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828363.111646:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787828363.111652:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787828363.111659:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c7016d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646722176:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787828363.111666:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787828363.111673:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828360/real 1787828363] req@ffff8d30c7014b40 x1874670646722048/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828385 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787828363.111697:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787828363.111701:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787828363.111707:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787828363.111712:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787828363.111714:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828363.111718:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828363.111720:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787828363.111729:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c7014b40 x1874670646722048/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828385 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787828363.111738:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828363.111741:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787828363.111743:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828363.111745:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828363.111747:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828363.111749:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787828363.111751:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787828363.111754:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828363.111759:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787828363.111762:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787828363.111765:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c7014b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646722048:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787828363.111771:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787828363.111776:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828360/real 1787828363] req@ffff8d30c70152c0 x1874670646721920/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828385 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787828363.111786:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787828363.111787:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787828363.111791:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787828363.111793:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787828363.111794:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828363.111796:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828363.111798:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787828363.111803:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c70152c0 x1874670646721920/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828385 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787828363.111812:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828363.111815:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787828363.111818:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828363.111821:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828363.111823:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828363.111824:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787828363.111827:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787828363.111830:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828363.111835:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787828363.111837:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787828363.111841:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c70152c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646721920:192.168.203.143@tcp:250:kworker.0 00000100:00000001:0.0:1787828363.111846:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828363.111850:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828363.111852:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787828363.111856:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787828363.111864:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787828363.111868:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787828363.111870:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787828363.111875:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787828363.111877:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787828363.111881:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787828363.111883:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828363.111886:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828363.111887:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787828363.111889:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787828363.111892:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787828363.111893:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787828363.111895:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787828363.111897:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787828363.111898:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787828363.111901:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787828363.111908:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828363.111911:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828363.111913:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787828363.111914:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787828363.111917:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787828363.111920:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787828363.111921:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787828363.111923:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787828363.111924:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787828363.111926:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787828363.111928:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828363.111930:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828363.111932:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787828363.111933:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787828363.111936:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787828363.111937:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787828363.111938:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787828363.111941:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787828363.111942:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787828363.111944:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787828363.111946:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828363.111949:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828363.111957:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787828363.111960:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828363.111962:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787828363.111965:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828364.006679:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828364.006684:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787828364.006741:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1787828364.006744:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1787828364.006748:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828365.030745:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828365.030751:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787828365.030807:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787828365.030811:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787828365.030814:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787828365.158824:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828365.158835:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1787828365.158839:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787828365.158843:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787828365.158865:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787828365.158873:0:40701:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 3165 00000100:00080000:2.0:1787828365.158885:0:40701:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787828365.158888:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828365.158891:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787828365.158895:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787828365.158898:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787828365.158902:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:2.0:1787828365.158904:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:2.0:1787828365.158907:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828365.158909:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:2.0:1787828365.158938:0:40701:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787828365.158942:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828365.158945:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787828365.158948:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828365.158952:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828365.158955:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:2.0:1787828365.158959:0:40701:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828365.158968:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787828365.158970:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828365.158974:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787828365.158993:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787828365.159010:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828365.159025:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787828365.159041:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2c000 x1874670646722432/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787828365.159055:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787828365.159059:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828365.159078:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787828365.159082:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828365.159086:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1787828365.159088:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787828365.159090:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787828365.159093:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787828365.159096:0:40701:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3165 00000100:00080000:2.0:1787828365.159117:0:40701:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787828365.159120:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828365.159121:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787828365.159125:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787828365.159127:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787828365.159129:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000020:00000001:2.0:1787828365.159133:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:2.0:1787828365.159136:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828365.159138:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:2.0:1787828365.159142:0:40701:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787828365.159144:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828365.159148:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:2.0:1787828365.159148:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787828365.159154:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787828365.159156:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 02000000:00000001:2.0:1787828365.159160:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828365.159162:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1787828365.159165:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1787828365.159167:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787828365.159172:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000008:00100000:2.0:1787828365.159173:0:40701:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 02000000:00000001:0.0:1787828365.159177:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000008:00000001:2.0:1787828365.159179:0:40701:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828365.159183:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787828365.159184:0:40701:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828365.159192:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828365.159195:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787828365.159199:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828365.159202:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646722432:192.168.203.143@tcp:250:kworker.0 00000100:00000001:2.0:1787828365.159203:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787828365.159207:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787828365.159212:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828365.159213:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787828365.159218:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787828365.159221:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828365.159223:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828365.159225:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1787828365.159234:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787828365.159235:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2da40 x1874670646722560/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787828365.159248:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787828365.159250:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828365.159252:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787828365.159255:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828365.159257:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417868288 : -126234291683328 : ffff8d30c9e2c000) 00000100:00080000:2.0:1787828365.159261:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1787828365.159264:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00080000:2.0:1787828365.159269:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1787828365.159273:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787828365.159274:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787828365.159278:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787828365.159281:0:40701:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3165 00000100:00080000:2.0:1787828365.159286:0:40701:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787828365.159289:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828365.159291:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787828365.159294:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787828365.159296:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787828365.159298:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000020:00000001:2.0:1787828365.159301:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:2.0:1787828365.159304:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828365.159305:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:2.0:1787828365.159313:0:40701:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000800:00000100:0.0:1787828365.159315:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:2.0:1787828365.159319:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828365.159324:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787828365.159328:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787828365.159331:0:2390:0:(events.c:34:request_out_callback()) Process entered 02000000:00000001:2.0:1787828365.159331:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828365.159337:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1787828365.159339:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000008:00100000:2.0:1787828365.159341:0:40701:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00000001:0.0:1787828365.159345:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:2.0:1787828365.159347:0:40701:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828365.159350:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1787828365.159352:0:40701:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828365.159360:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828365.159361:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787828365.159364:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828365.159370:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4297839360 00000100:00000001:2.0:1787828365.159370:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787828365.159374:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787828365.159378:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828365.159384:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 00000100:00000001:2.0:1787828365.159385:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828365.159395:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4297839360 00000100:00100000:2.0:1787828365.159396:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2ed00 x1874670646722688/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787828365.159405:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 00000100:00000001:2.0:1787828365.159411:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787828365.159413:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787828365.159416:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828365.159420:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828365.159422:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828365.159426:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00080000:2.0:1787828365.159427:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828365.159434:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1787828365.159438:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00100000:0.0:1787828365.159439:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828365/real 1787828365] req@ffff8d30c9e2c000 x1874670646722432/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828390 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828365.159440:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787828365.159448:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787828365.159455:0:40701:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3165 00000400:00000001:0.0:1787828365.159463:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00080000:2.0:1787828365.159463:0:40701:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000400:00000001:0.0:1787828365.159468:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1787828365.159469:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828365.159474:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:0.0:1787828365.159476:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000020:00000001:2.0:1787828365.159477:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787828365.159481:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:0.0:1787828365.159482:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000020:00000001:2.0:1787828365.159485:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:0.0:1787828365.159487:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:2.0:1787828365.159493:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:0.0:1787828365.159494:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828365.159498:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828365.159500:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00100000:0.0:1787828365.159510:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2c000 x1874670646722432/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828390 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787828365.159528:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:2.0:1787828365.159539:0:40701:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787828365.159545:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828365.159546:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828365.159553:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 02000000:00000001:2.0:1787828365.159553:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787828365.159557:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787828365.159559:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:2.0:1787828365.159562:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828365.159564:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828365.159568:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828365.159568:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1787828365.159574:0:40701:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00000001:0.0:1787828365.159575:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787828365.159579:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000008:00000001:2.0:1787828365.159581:0:40701:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787828365.159585:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00000001:2.0:1787828365.159587:0:40701:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787828365.159596:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 02000000:00000001:2.0:1787828365.159597:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787828365.159629:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828365.159631:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787828365.159633:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787828365.159636:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828365.159643:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828365.159650:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1787828365.159650:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2f480 x1874670646722816/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787828365.159662:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646722432:192.168.203.143@tcp:250:kworker.0 00000100:00000001:2.0:1787828365.159670:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787828365.159673:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828365.159676:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828365.159679:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828365.159682:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828365.159683:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787828365.159689:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787828365.159695:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787828365.159697:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787828365.159700:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787828365.159704:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787828365.159707:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787828365.159712:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787828365.159714:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828365.159717:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828365.159724:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787828365.159727:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828365.159730:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787828365.159732:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787828365.159733:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1787828365.159735:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787828365.159737:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828365.159739:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787828365.159741:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787828365.159744:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828365.159748:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2da40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646722560:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787828365.159752:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787828365.159755:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787828365.159757:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828365.159759:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1787828365.159763:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828365.159777:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787828365.159779:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417875008 : -126234291676608 : ffff8d30c9e2da40) 00000100:00000001:0.0:1787828365.159785:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1787828365.159806:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:0.0:1787828365.159812:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1787828365.159815:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828365.159817:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828365.159820:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787828365.159823:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828365.159829:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4297838103 00000100:00000001:0.0:1787828365.159833:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4297839360 00000100:00000001:0.0:1787828365.159836:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4297838103 00000100:00000001:0.0:1787828365.159838:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4297839360 00000100:00000001:0.0:1787828365.159841:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787828365.159844:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828365.159846:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787828365.159859:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828365/real 1787828365] req@ffff8d30c9e2da40 x1874670646722560/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828390 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787828365.159871:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787828365.159872:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787828365.159875:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787828365.159877:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787828365.159879:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828365.159881:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828365.159883:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787828365.159888:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2da40 x1874670646722560/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828390 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787828365.159898:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828365.159901:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787828365.159904:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828365.159906:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828365.159907:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828365.159910:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787828365.159912:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787828365.159916:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828365.159921:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787828365.159924:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787828365.159929:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2da40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646722560:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787828365.159933:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1787828365.159935:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787828365.159936:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828365.159939:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787828365.159941:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787828365.159944:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828365.159948:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646722688:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787828365.159951:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787828365.159954:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787828365.159956:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828365.159958:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1787828365.159962:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828365.159969:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787828365.159971:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417879808 : -126234291671808 : ffff8d30c9e2ed00) 00000100:00000001:0.0:1787828365.159975:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1787828365.159992:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:0.0:1787828365.159999:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1787828365.160002:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828365.160004:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828365.160006:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787828365.160010:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828365.160015:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4297838103 00000100:00000001:0.0:1787828365.160018:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4297839360 00000100:00000001:0.0:1787828365.160021:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4297838103 00000100:00000001:0.0:1787828365.160024:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4297839360 00000100:00000001:0.0:1787828365.160026:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787828365.160029:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828365.160031:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787828365.160037:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828365/real 1787828365] req@ffff8d30c9e2ed00 x1874670646722688/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828390 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787828365.160054:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787828365.160056:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787828365.160060:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787828365.160061:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787828365.160063:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828365.160065:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828365.160067:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787828365.160072:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2ed00 x1874670646722688/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828390 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787828365.160079:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828365.160082:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787828365.160085:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828365.160087:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828365.160089:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828365.160091:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787828365.160092:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787828365.160096:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828365.160100:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787828365.160103:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787828365.160107:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646722688:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787828365.160113:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1787828365.160114:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787828365.160116:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828365.160118:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787828365.160121:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787828365.160123:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828365.160127:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646722816:192.168.203.143@tcp:8:kworker.0 00000100:00000001:0.0:1787828365.160131:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787828365.160133:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787828365.160136:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828365.160138:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1787828365.160142:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828365.160148:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787828365.160149:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417881728 : -126234291669888 : ffff8d30c9e2f480) 00000100:00000001:0.0:1787828365.160153:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1787828365.160168:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:0.0:1787828365.160174:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1787828365.160177:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828365.160179:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828365.160181:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787828365.160184:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828365.160188:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4297838116 00000100:00000001:0.0:1787828365.160191:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4297839360 00000100:00000001:0.0:1787828365.160194:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4297838116 00000100:00000001:0.0:1787828365.160196:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4297839360 00000100:00000001:0.0:1787828365.160199:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787828365.160201:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828365.160203:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787828365.160208:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828365/real 1787828365] req@ffff8d30c9e2f480 x1874670646722816/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828390 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787828365.160219:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787828365.160220:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787828365.160224:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787828365.160226:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787828365.160227:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828365.160229:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828365.160231:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787828365.160234:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2f480 x1874670646722816/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828390 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787828365.160242:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828365.160245:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787828365.160249:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828365.160250:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828365.160252:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828365.160254:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787828365.160256:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787828365.160265:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828365.160270:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787828365.160272:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787828365.160276:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646722816:192.168.203.143@tcp:8:kworker.0 00000100:00000001:0.0:1787828365.160281:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828365.160284:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828365.160285:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787828365.160288:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787828365.160290:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787828365.160293:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787828365.160294:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787828365.160297:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787828365.160299:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787828365.160303:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787828365.160305:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828365.160307:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828365.160308:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787828365.160309:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787828365.160311:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787828365.160312:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787828365.160314:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787828365.160316:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787828365.160317:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787828365.160321:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787828365.160323:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828365.160325:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828365.160326:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787828365.160328:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787828365.160330:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787828365.160331:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787828365.160333:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787828365.160334:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787828365.160336:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787828365.160338:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787828365.160339:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828365.160341:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828365.160346:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787828365.160348:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828365.160350:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787828365.160352:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828366.054721:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828366.054731:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787828366.054796:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787828366.054801:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787828366.054804:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828367.078741:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828367.078763:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787828367.078863:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1787828367.078869:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1787828367.078873:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828368.102722:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828368.102734:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787828368.102864:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787828368.102871:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787828368.102875:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828369.126739:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828369.126747:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787828369.126859:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787828369.126867:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787828369.126873:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828370.150745:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828370.150757:0:815:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1787828370.150760:0:815:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1787828370.150764:0:815:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828370.150771:0:815:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1787828370.150774:0:815:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828370.150793:0:815:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828370.150797:0:815:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787828370.150891:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1787828370.150895:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1787828370.150898:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787828370.278678:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828370.278687:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1787828370.278689:0:38:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787828370.278692:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787828370.278696:0:38:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787828370.278702:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 3170 00000100:00080000:2.0:1787828370.278727:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787828370.278729:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828370.278731:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787828370.278735:0:38:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787828370.278737:0:38:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787828370.278739:0:38:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:2.0:1787828370.278749:0:38:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:2.0:1787828370.278751:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828370.278752:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:2.0:1787828370.278756:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787828370.278758:0:38:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828370.278761:0:38:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787828370.278763:0:38:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828370.278766:0:38:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828370.278769:0:38:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:2.0:1787828370.278771:0:38:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828370.278781:0:38:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787828370.278782:0:38:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828370.278785:0:38:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787828370.278787:0:38:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787828370.278791:0:38:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828370.278801:0:38:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787828370.278809:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2e580 x1874670646722944/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787828370.278817:0:38:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787828370.278821:0:38:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828370.278834:0:38:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787828370.278837:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828370.278840:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1787828370.278841:0:38:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787828370.278842:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787828370.278844:0:38:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787828370.278846:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3170 00000100:00080000:2.0:1787828370.278848:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787828370.278850:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828370.278851:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787828370.278853:0:38:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787828370.278854:0:38:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787828370.278856:0:38:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000020:00000001:2.0:1787828370.278858:0:38:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:2.0:1787828370.278860:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828370.278861:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:2.0:1787828370.278864:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787828370.278866:0:38:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828370.278868:0:38:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:0.0:1787828370.278874:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:2.0:1787828370.278874:0:38:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828370.278879:0:38:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828370.278881:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1787828370.278881:0:38:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1787828370.278885:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000008:00100000:2.0:1787828370.278888:0:38:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00000001:0.0:1787828370.278889:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787828370.278892:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:2.0:1787828370.278893:0:38:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828370.278898:0:38:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828370.278899:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787828370.278901:0:38:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787828370.278903:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828370.278904:0:38:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828370.278909:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828370.278910:0:38:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787828370.278914:0:38:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787828370.278917:0:38:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828370.278920:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646722944:192.168.203.143@tcp:250:kworker.0 00000100:00000001:2.0:1787828370.278921:0:38:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828370.278931:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00100000:2.0:1787828370.278931:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2c3c0 x1874670646723072/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:0.0:1787828370.278939:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:2.0:1787828370.278940:0:38:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787828370.278944:0:38:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828370.278960:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828370.278962:0:38:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828370.278965:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00080000:2.0:1787828370.278966:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828370.278970:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1787828370.278971:0:38:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787828370.278972:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 02000000:00000001:0.0:1787828370.278973:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828370.278975:0:38:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787828370.278979:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3170 00000100:00080000:2.0:1787828370.278982:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787828370.278983:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828370.278984:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787828370.278986:0:38:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787828370.278986:0:38:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787828370.278988:0:38:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000020:00000001:2.0:1787828370.278989:0:38:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:2.0:1787828370.278991:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787828370.278992:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787828370.278993:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:0.0:1787828370.278997:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417877888 : -126234291673728 : ffff8d30c9e2e580) 00000100:00080000:2.0:1787828370.278998:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787828370.279003:0:38:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828370.279005:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 02000000:00000001:2.0:1787828370.279005:0:38:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787828370.279009:0:38:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828370.279012:0:38:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828370.279013:0:38:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1787828370.279014:0:38:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1787828370.279016:0:38:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828370.279017:0:38:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828370.279020:0:38:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787828370.279022:0:38:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828370.279023:0:38:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787828370.279023:0:38:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787828370.279025:0:38:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828370.279028:0:38:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787828370.279030:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2f0c0 x1874670646723200/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787828370.279035:0:38:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787828370.279037:0:38:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828370.279038:0:38:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787828370.279040:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828370.279042:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1787828370.279043:0:38:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787828370.279045:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:0.0:1787828370.279048:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:2.0:1787828370.279049:0:38:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787828370.279054:0:38:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3170 00000100:00080000:2.0:1787828370.279057:0:38:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787828370.279058:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828370.279059:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787828370.279061:0:38:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787828370.279062:0:38:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:0.0:1787828370.279064:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000020:00000001:2.0:1787828370.279064:0:38:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000020:00000001:2.0:1787828370.279068:0:38:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:2.0:1787828370.279069:0:38:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787828370.279071:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828370.279071:0:38:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:0.0:1787828370.279077:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787828370.279077:0:38:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787828370.279083:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1787828370.279084:0:38:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828370.279088:0:38:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787828370.279089:0:38:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828370.279091:0:38:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828370.279092:0:38:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1787828370.279093:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000008:00100000:2.0:1787828370.279094:0:38:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000008:00000001:2.0:1787828370.279100:0:38:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828370.279102:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4297844480 00000100:00000001:2.0:1787828370.279102:0:38:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828370.279106:0:38:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787828370.279108:0:38:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828370.279109:0:38:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787828370.279110:0:38:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787828370.279111:0:38:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828370.279115:0:38:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828370.279116:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 00000100:00100000:2.0:1787828370.279119:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2e940 x1874670646723328/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787828370.279126:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4297844480 00000100:00000001:2.0:1787828370.279126:0:38:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787828370.279130:0:38:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828370.279133:0:38:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828370.279135:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1787828370.279142:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787828370.279146:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828370.279150:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787828370.279156:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828370/real 1787828370] req@ffff8d30c9e2e580 x1874670646722944/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828395 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787828370.279175:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787828370.279176:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787828370.279179:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787828370.279181:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787828370.279182:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828370.279184:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828370.279186:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787828370.279189:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2e580 x1874670646722944/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828395 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787828370.279194:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828370.279213:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787828370.279215:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828370.279217:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828370.279218:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828370.279220:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787828370.279221:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787828370.279224:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828370.279227:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787828370.279229:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787828370.279231:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646722944:192.168.203.143@tcp:250:kworker.0 00000100:00000001:0.0:1787828370.279236:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828370.279237:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828370.279239:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787828370.279241:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787828370.279244:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787828370.279246:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787828370.279247:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787828370.279251:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787828370.279252:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787828370.279255:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787828370.279256:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828370.279258:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828370.279265:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787828370.279267:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828370.279268:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787828370.279270:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787828370.279271:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1787828370.279272:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787828370.279274:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828370.279275:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787828370.279277:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787828370.279279:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828370.279281:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646723072:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787828370.279284:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787828370.279286:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787828370.279287:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828370.279289:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1787828370.279291:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828370.279296:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787828370.279297:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417869248 : -126234291682368 : ffff8d30c9e2c3c0) 00000100:00000001:0.0:1787828370.279301:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1787828370.279328:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:0.0:1787828370.279341:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1787828370.279344:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828370.279346:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828370.279348:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787828370.279351:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828370.279356:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4297843222 00000100:00000001:0.0:1787828370.279360:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4297844480 00000100:00000001:0.0:1787828370.279362:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4297843222 00000100:00000001:0.0:1787828370.279365:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4297844480 00000100:00000001:0.0:1787828370.279368:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787828370.279384:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828370.279386:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787828370.279391:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828370/real 1787828370] req@ffff8d30c9e2c3c0 x1874670646723072/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828395 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787828370.279411:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787828370.279413:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787828370.279416:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787828370.279417:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787828370.279419:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828370.279420:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828370.279422:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787828370.279426:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2c3c0 x1874670646723072/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828395 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787828370.279431:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828370.279433:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787828370.279434:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828370.279435:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828370.279436:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828370.279437:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787828370.279438:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787828370.279440:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828370.279442:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787828370.279443:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787828370.279445:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646723072:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787828370.279448:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1787828370.279449:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787828370.279450:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828370.279452:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787828370.279453:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787828370.279455:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828370.279457:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646723200:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787828370.279470:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787828370.279471:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787828370.279472:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828370.279473:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1787828370.279489:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828370.279494:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787828370.279495:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417880768 : -126234291670848 : ffff8d30c9e2f0c0) 00000100:00000001:0.0:1787828370.279498:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1787828370.279507:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:0.0:1787828370.279510:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1787828370.279528:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828370.279529:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828370.279531:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787828370.279533:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828370.279536:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4297843222 00000100:00000001:0.0:1787828370.279538:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4297844480 00000100:00000001:0.0:1787828370.279540:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4297843222 00000100:00000001:0.0:1787828370.279541:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4297844480 00000100:00000001:0.0:1787828370.279543:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787828370.279544:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828370.279546:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787828370.279549:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828370/real 1787828370] req@ffff8d30c9e2f0c0 x1874670646723200/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828395 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787828370.279555:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787828370.279556:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787828370.279557:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787828370.279558:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787828370.279559:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828370.279561:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828370.279562:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787828370.279565:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2f0c0 x1874670646723200/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828395 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787828370.279570:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828370.279572:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787828370.279573:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828370.279574:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828370.279575:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828370.279577:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787828370.279593:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787828370.279595:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828370.279597:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787828370.279599:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787828370.279601:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646723200:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787828370.279604:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1787828370.279605:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787828370.279607:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828370.279609:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787828370.279611:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787828370.279614:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828370.279617:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646723328:192.168.203.143@tcp:8:kworker.0 00000100:00000001:0.0:1787828370.279622:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787828370.279624:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787828370.279626:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828370.279628:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1787828370.279701:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828370.279707:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787828370.279708:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417878848 : -126234291672768 : ffff8d30c9e2e940) 00000100:00000001:0.0:1787828370.279711:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1787828370.279720:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:0.0:1787828370.279740:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1787828370.279742:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828370.279744:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828370.279745:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787828370.279748:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828370.279750:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4297843236 00000100:00000001:0.0:1787828370.279752:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4297844480 00000100:00000001:0.0:1787828370.279754:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4297843236 00000100:00000001:0.0:1787828370.279756:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4297844480 00000100:00000001:0.0:1787828370.279757:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787828370.279759:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828370.279760:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787828370.279763:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828370/real 1787828370] req@ffff8d30c9e2e940 x1874670646723328/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828395 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787828370.279769:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787828370.279770:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787828370.279771:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787828370.279772:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787828370.279774:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828370.279775:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828370.279776:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787828370.279779:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2e940 x1874670646723328/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828395 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787828370.279784:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828370.279785:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787828370.279787:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828370.279789:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828370.279790:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828370.279791:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787828370.279792:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787828370.279794:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828370.279796:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787828370.279797:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787828370.279800:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646723328:192.168.203.143@tcp:8:kworker.0 00000100:00000001:0.0:1787828370.279803:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828370.279804:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828370.279805:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787828370.279807:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787828370.279808:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787828370.279809:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787828370.279810:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787828370.279812:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787828370.279813:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787828370.279815:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787828370.279816:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828370.279817:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828370.279818:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787828370.279819:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787828370.279821:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787828370.279822:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787828370.279823:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787828370.279824:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787828370.279825:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787828370.279827:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787828370.279828:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828370.279829:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828370.279830:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787828370.279830:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787828370.279831:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787828370.279832:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787828370.279833:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787828370.279834:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787828370.279835:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787828370.279837:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787828370.279838:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828370.279839:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828370.279842:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787828370.279843:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828370.279844:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787828370.279845:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828371.174723:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828371.174729:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787828371.174788:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787828371.174792:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787828371.174795:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828372.198682:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828372.198690:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787828372.198759:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787828372.198763:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787828372.198766:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828373.222719:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828373.222725:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787828373.222796:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1787828373.222802:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1787828373.222807:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828374.246737:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828374.246744:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787828374.246818:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787828374.246823:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787828374.246827:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828375.270672:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828375.270678:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787828375.270748:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787828375.270752:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787828375.270755:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1787828375.398899:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787828375.398909:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787828375.398916:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1787828375.398923:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787828375.398930:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828375.398939:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787828375.398956:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787828375.398963:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828375.398971:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c7014f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646723456:192.168.203.143@tcp:250:kworker.0 00000100:00000001:0.0:1787828375.398980:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787828375.398986:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787828375.398989:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828375.398993:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1787828375.398999:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828375.399014:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787828375.399018:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369539328 : -126234340012288 : ffff8d30c7014f00) 00000100:00000001:0.0:1787828375.399022:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1787828375.399063:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:0.0:1787828375.399077:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1787828375.399083:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828375.399087:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828375.399092:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787828375.399099:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828375.399105:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4297849600 00000100:00000001:0.0:1787828375.399115:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1787828375.399123:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4297849600 00000100:00000001:0.0:1787828375.399129:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1787828375.399134:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787828375.399150:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828375.399154:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787828375.399163:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828375/real 1787828375] req@ffff8d30c7014f00 x1874670646723456/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828400 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787828375.399181:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787828375.399185:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787828375.399191:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787828375.399196:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787828375.399200:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828375.399206:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828375.399209:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787828375.399218:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c7014f00 x1874670646723456/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828400 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787828375.399244:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828375.399249:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787828375.399254:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828375.399257:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828375.399262:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828375.399264:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787828375.399268:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787828375.399273:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828375.399279:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787828375.399283:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787828375.399289:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c7014f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646723456:192.168.203.143@tcp:250:kworker.0 00000100:00000001:0.0:1787828375.399298:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828375.399304:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828375.399308:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787828375.399323:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787828375.399328:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787828375.399332:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787828375.399335:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787828375.399341:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787828375.399345:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787828375.399349:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787828375.399353:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828375.399355:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828375.399364:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787828375.399367:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828375.399370:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787828375.399374:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787828375.399377:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1787828375.399393:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787828375.399396:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828375.399401:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787828375.399406:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787828375.399412:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828375.399419:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c70143c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646723584:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787828375.399424:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787828375.399425:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787828375.399426:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828375.399427:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1787828375.399430:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828375.399436:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787828375.399438:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369536448 : -126234340015168 : ffff8d30c70143c0) 00000100:00000001:0.0:1787828375.399457:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1787828375.399472:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:0.0:1787828375.399476:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1787828375.399478:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828375.399479:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828375.399481:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787828375.399483:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828375.399487:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4297848342 00000100:00000001:0.0:1787828375.399489:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4297849600 00000100:00000001:0.0:1787828375.399491:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4297848342 00000100:00000001:0.0:1787828375.399493:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4297849600 00000100:00000001:0.0:1787828375.399494:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787828375.399496:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828375.399497:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787828375.399501:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828375/real 1787828375] req@ffff8d30c70143c0 x1874670646723584/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828400 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787828375.399507:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787828375.399508:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787828375.399509:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787828375.399511:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787828375.399512:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828375.399514:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828375.399515:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787828375.399518:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c70143c0 x1874670646723584/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828400 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787828375.399523:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828375.399525:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787828375.399526:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828375.399527:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828375.399528:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828375.399530:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787828375.399531:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787828375.399533:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828375.399535:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787828375.399536:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787828375.399539:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c70143c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646723584:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787828375.399541:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1787828375.399543:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787828375.399544:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828375.399545:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787828375.399547:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787828375.399548:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828375.399551:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c7015680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646723712:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787828375.399553:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787828375.399555:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787828375.399556:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828375.399557:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1787828375.399559:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828375.399563:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787828375.399564:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369541248 : -126234340010368 : ffff8d30c7015680) 00000100:00000001:0.0:1787828375.399566:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1787828375.399576:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:0.0:1787828375.399580:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1787828375.399583:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828375.399585:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828375.399586:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787828375.399589:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828375.399593:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4297848342 00000100:00000001:0.0:1787828375.399596:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4297849600 00000100:00000001:0.0:1787828375.399648:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4297848343 00000100:00000001:0.0:1787828375.399686:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4297849600 00000100:00000001:0.0:1787828375.399689:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787828375.399690:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828375.399692:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787828375.399697:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828375/real 1787828375] req@ffff8d30c7015680 x1874670646723712/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828400 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787828375.399706:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787828375.399708:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787828375.399712:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787828375.399714:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787828375.399716:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828375.399718:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828375.399720:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787828375.399724:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c7015680 x1874670646723712/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828400 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787828375.399730:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828375.399733:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787828375.399736:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828375.399738:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828375.399739:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828375.399740:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787828375.399742:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787828375.399745:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828375.399748:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787828375.399750:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787828375.399753:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c7015680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646723712:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787828375.399759:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828375.399761:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828375.399762:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787828375.399763:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787828375.399766:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787828375.399767:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787828375.399768:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787828375.399770:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787828375.399771:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787828375.399773:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787828375.399774:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828375.399775:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828375.399776:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787828375.399777:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787828375.399778:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787828375.399779:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787828375.399780:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787828375.399781:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787828375.399782:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787828375.399783:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787828375.399784:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828375.399785:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828375.399788:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787828375.399789:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828375.399791:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787828375.399792:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787828375.399793:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1787828375.399794:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787828375.399795:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828375.399796:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787828375.399798:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787828375.399799:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828375.399802:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c7015a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646723840:192.168.203.143@tcp:8:kworker.0 00000100:00000001:0.0:1787828375.399804:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787828375.399806:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787828375.399807:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828375.399809:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1787828375.399811:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828375.399816:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787828375.399817:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369542208 : -126234340009408 : ffff8d30c7015a40) 00000100:00000001:0.0:1787828375.399820:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1787828375.399830:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:0.0:1787828375.399833:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1787828375.399835:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828375.399836:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828375.399838:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787828375.399840:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828375.399842:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4297848356 00000100:00000001:0.0:1787828375.399844:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4297849600 00000100:00000001:0.0:1787828375.399846:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4297848356 00000100:00000001:0.0:1787828375.399848:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4297849600 00000100:00000001:0.0:1787828375.399850:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787828375.399851:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828375.399852:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787828375.399855:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828375/real 1787828375] req@ffff8d30c7015a40 x1874670646723840/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828400 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787828375.399860:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787828375.399861:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787828375.399863:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787828375.399864:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787828375.399865:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828375.399866:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828375.399868:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787828375.399870:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c7015a40 x1874670646723840/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828400 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787828375.399875:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828375.399877:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787828375.399879:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828375.399880:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828375.399881:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828375.399882:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787828375.399884:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787828375.399886:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828375.399888:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787828375.399890:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787828375.399894:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c7015a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646723840:192.168.203.143@tcp:8:kworker.0 00000100:00000001:0.0:1787828375.399898:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828375.399900:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828375.399901:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787828375.399903:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787828375.399905:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787828375.399907:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787828375.399908:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787828375.399910:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787828375.399912:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787828375.399914:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787828375.399915:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828375.399917:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828375.399921:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787828375.399923:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828375.399925:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787828375.399927:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828376.294722:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828376.294732:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787828376.294869:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1787828376.294875:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1787828376.294880:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828377.318674:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828377.318679:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787828377.318725:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787828377.318728:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787828377.318732:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828378.342721:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828378.342734:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787828378.342886:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787828378.342891:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787828378.342895:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828379.366722:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828379.366749:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787828379.366849:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1787828379.366855:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1787828379.366858:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828380.390667:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828380.390676:0:815:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1787828380.390678:0:815:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1787828380.390681:0:815:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828380.390686:0:815:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1787828380.390688:0:815:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828380.390690:0:815:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828380.390693:0:815:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787828380.390732:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787828380.390736:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787828380.390739:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1787828380.518943:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787828380.518948:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787828380.518953:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1787828380.518957:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787828380.518961:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828380.518967:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787828380.518971:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787828380.518975:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828380.518982:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c70161c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646723968:192.168.203.143@tcp:250:kworker.0 00000100:00000001:0.0:1787828380.518988:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787828380.518991:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787828380.518995:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828380.518998:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1787828380.519004:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828380.519018:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787828380.519021:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369544128 : -126234340007488 : ffff8d30c70161c0) 00000100:00000001:0.0:1787828380.519027:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1787828380.519074:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:0.0:1787828380.519086:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1787828380.519092:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828380.519095:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828380.519108:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787828380.519114:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828380.519120:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4297854720 00000100:00000001:0.0:1787828380.519131:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1787828380.519140:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4297854720 00000100:00000001:0.0:1787828380.519147:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1787828380.519154:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787828380.519158:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828380.519174:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787828380.519181:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828380/real 1787828380] req@ffff8d30c70161c0 x1874670646723968/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828405 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787828380.519198:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787828380.519202:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787828380.519207:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787828380.519211:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787828380.519215:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828380.519218:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828380.519222:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787828380.519230:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c70161c0 x1874670646723968/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828405 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787828380.519249:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828380.519255:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787828380.519260:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828380.519264:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828380.519266:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828380.519270:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787828380.519273:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787828380.519276:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828380.519280:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787828380.519282:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787828380.519285:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c70161c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646723968:192.168.203.143@tcp:250:kworker.0 00000100:00000001:0.0:1787828380.519290:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828380.519292:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828380.519293:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787828380.519295:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787828380.519297:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787828380.519299:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787828380.519301:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787828380.519304:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787828380.519306:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787828380.519319:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787828380.519320:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828380.519322:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828380.519327:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787828380.519329:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828380.519330:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787828380.519332:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787828380.519333:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1787828380.519334:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787828380.519335:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828380.519336:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787828380.519338:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787828380.519339:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828380.519342:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c7016580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646724096:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787828380.519344:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787828380.519345:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787828380.519347:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828380.519348:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1787828380.519350:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828380.519354:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787828380.519355:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369545088 : -126234340006528 : ffff8d30c7016580) 00000100:00000001:0.0:1787828380.519359:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1787828380.519366:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:0.0:1787828380.519369:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1787828380.519371:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828380.519372:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828380.519373:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787828380.519375:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828380.519378:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4297853462 00000100:00000001:0.0:1787828380.519381:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4297854720 00000100:00000001:0.0:1787828380.519382:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4297853462 00000100:00000001:0.0:1787828380.519384:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4297854720 00000100:00000001:0.0:1787828380.519386:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787828380.519387:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828380.519388:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787828380.519391:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828380/real 1787828380] req@ffff8d30c7016580 x1874670646724096/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828405 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787828380.519397:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787828380.519398:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787828380.519399:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787828380.519401:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787828380.519402:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828380.519403:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828380.519417:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787828380.519430:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c7016580 x1874670646724096/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828405 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787828380.519440:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828380.519442:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787828380.519444:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828380.519445:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828380.519446:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828380.519447:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787828380.519448:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787828380.519450:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828380.519452:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787828380.519453:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787828380.519455:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c7016580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646724096:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787828380.519458:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1787828380.519459:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787828380.519460:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828380.519462:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787828380.519463:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787828380.519465:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828380.519466:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c7015a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646724224:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787828380.519469:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787828380.519470:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787828380.519471:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828380.519473:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1787828380.519475:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828380.519478:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787828380.519478:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369542208 : -126234340009408 : ffff8d30c7015a40) 00000100:00000001:0.0:1787828380.519481:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1787828380.519488:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:0.0:1787828380.519496:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1787828380.519497:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828380.519498:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828380.519500:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787828380.519501:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828380.519503:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4297853462 00000100:00000001:0.0:1787828380.519505:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4297854720 00000100:00000001:0.0:1787828380.519507:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4297853462 00000100:00000001:0.0:1787828380.519509:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4297854720 00000100:00000001:0.0:1787828380.519510:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787828380.519511:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828380.519512:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787828380.519515:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828380/real 1787828380] req@ffff8d30c7015a40 x1874670646724224/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828405 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787828380.519520:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787828380.519521:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787828380.519522:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787828380.519523:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787828380.519524:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828380.519525:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828380.519526:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787828380.519528:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c7015a40 x1874670646724224/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828405 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787828380.519533:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828380.519535:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787828380.519537:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828380.519538:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828380.519539:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828380.519540:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787828380.519541:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787828380.519542:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828380.519544:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787828380.519546:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787828380.519548:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c7015a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646724224:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787828380.519551:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1787828380.519552:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787828380.519553:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828380.519554:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787828380.519555:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787828380.519557:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828380.519558:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c7015680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646724352:192.168.203.143@tcp:8:kworker.0 00000100:00000001:0.0:1787828380.519560:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787828380.519561:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787828380.519562:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828380.519563:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1787828380.519565:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828380.519568:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787828380.519569:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369541248 : -126234340010368 : ffff8d30c7015680) 00000100:00000001:0.0:1787828380.519571:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1787828380.519576:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:0.0:1787828380.519578:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1787828380.519580:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828380.519581:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828380.519582:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787828380.519583:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828380.519586:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4297853475 00000100:00000001:0.0:1787828380.519587:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4297854720 00000100:00000001:0.0:1787828380.519589:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4297853475 00000100:00000001:0.0:1787828380.519590:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4297854720 00000100:00000001:0.0:1787828380.519592:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787828380.519593:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828380.519594:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787828380.519597:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828380/real 1787828380] req@ffff8d30c7015680 x1874670646724352/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828405 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787828380.519634:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787828380.519635:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787828380.519637:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787828380.519638:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787828380.519639:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828380.519640:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828380.519642:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787828380.519645:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c7015680 x1874670646724352/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828405 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787828380.519649:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828380.519651:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787828380.519653:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828380.519654:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828380.519655:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828380.519656:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787828380.519657:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787828380.519658:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828380.519660:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787828380.519662:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787828380.519664:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c7015680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646724352:192.168.203.143@tcp:8:kworker.0 00000100:00000001:0.0:1787828380.519666:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828380.519667:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828380.519668:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787828380.519669:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787828380.519671:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787828380.519672:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787828380.519673:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787828380.519674:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787828380.519675:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787828380.519677:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787828380.519678:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828380.519679:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828380.519680:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787828380.519681:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787828380.519682:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787828380.519683:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787828380.519684:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787828380.519685:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787828380.519685:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787828380.519687:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787828380.519687:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828380.519688:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828380.519689:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787828380.519690:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787828380.519691:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787828380.519691:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787828380.519692:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787828380.519693:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787828380.519694:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787828380.519695:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787828380.519696:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828380.519697:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828380.519699:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787828380.519700:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828380.519701:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787828380.519702:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828381.414694:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828381.414700:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787828381.414824:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787828381.414832:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787828381.414838:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828382.438661:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828382.438667:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787828382.438725:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1787828382.438728:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1787828382.438731:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828383.462684:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828383.462691:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787828383.462801:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787828383.462808:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787828383.462814:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828384.486712:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828384.486718:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787828384.486782:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787828384.486787:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787828384.486790:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828385.510692:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828385.510700:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787828385.510763:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1787828385.510767:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1787828385.510771:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787828385.638701:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828385.638709:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1787828385.638711:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787828385.638714:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787828385.638718:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787828385.638726:0:40701:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 3186 00000100:00080000:2.0:1787828385.638736:0:40701:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787828385.638740:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828385.638742:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787828385.638747:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787828385.638749:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787828385.638752:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:2.0:1787828385.638755:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:2.0:1787828385.638757:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828385.638759:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:2.0:1787828385.638764:0:40701:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787828385.638767:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828385.638778:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787828385.638781:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828385.638783:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828385.638785:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:2.0:1787828385.638787:0:40701:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828385.638797:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787828385.638799:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828385.638802:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787828385.638804:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787828385.638807:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828385.638816:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787828385.638824:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2c780 x1874670646724480/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787828385.638832:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787828385.638835:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828385.638847:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787828385.638849:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828385.638852:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1787828385.638853:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787828385.638854:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787828385.638856:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787828385.638858:0:40701:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3186 00000100:00080000:2.0:1787828385.638861:0:40701:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787828385.638862:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828385.638864:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787828385.638865:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787828385.638866:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787828385.638868:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000020:00000001:2.0:1787828385.638869:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:2.0:1787828385.638870:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828385.638871:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:2.0:1787828385.638873:0:40701:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787828385.638874:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828385.638875:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787828385.638879:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828385.638880:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828385.638881:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1787828385.638885:0:40701:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1787828385.638887:0:40701:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828385.638889:0:40701:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828385.638891:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787828385.638892:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828385.638893:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787828385.638894:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787828385.638896:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828385.638899:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787828385.638903:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2de00 x1874670646724608/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787828385.638909:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787828385.638910:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828385.638913:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787828385.638913:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787828385.638915:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1787828385.638919:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00080000:2.0:1787828385.638921:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1787828385.638922:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1787828385.638927:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787828385.638927:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:0.0:1787828385.638932:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787828385.638932:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 02000000:00000001:0.0:1787828385.638938:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:2.0:1787828385.638939:0:40701:0:(import.c:485:import_select_connection()) Process entered 02000000:00000001:0.0:1787828385.638941:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787828385.638946:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787828385.638946:0:40701:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3186 00000100:00100000:0.0:1787828385.638953:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646724480:192.168.203.143@tcp:250:kworker.0 00000100:00080000:2.0:1787828385.638954:0:40701:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787828385.638960:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828385.638964:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:0.0:1787828385.638965:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787828385.638970:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000020:00000001:2.0:1787828385.638970:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787828385.638972:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 02000000:00000001:0.0:1787828385.638974:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1787828385.638975:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 02000000:00000001:0.0:1787828385.638977:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000020:00000001:2.0:1787828385.638979:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:2.0:1787828385.638982:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828385.638983:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 02000000:00000001:0.0:1787828385.638984:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787828385.638987:0:40701:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787828385.638990:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828385.638991:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787828385.638994:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828385.638996:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828385.638997:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1787828385.638999:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000008:00100000:2.0:1787828385.639000:0:40701:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00000001:0.0:1787828385.639003:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417870208 : -126234291681408 : ffff8d30c9e2c780) 00000008:00000001:2.0:1787828385.639004:0:40701:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828385.639009:0:40701:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828385.639010:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 02000000:00000001:2.0:1787828385.639014:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787828385.639016:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828385.639018:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787828385.639020:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787828385.639021:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828385.639026:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787828385.639030:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2fc00 x1874670646724736/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787828385.639035:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787828385.639037:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828385.639039:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787828385.639041:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828385.639044:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1787828385.639046:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000800:00000100:0.0:1787828385.639048:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00080000:2.0:1787828385.639049:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787828385.639054:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787828385.639057:0:40701:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3186 00000100:00000001:0.0:1787828385.639061:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00080000:2.0:1787828385.639061:0:40701:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787828385.639063:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828385.639065:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:0.0:1787828385.639067:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000020:00000001:2.0:1787828385.639068:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787828385.639071:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:0.0:1787828385.639072:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1787828385.639073:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:0.0:1787828385.639076:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000020:00000001:2.0:1787828385.639077:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:2.0:1787828385.639080:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828385.639081:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:0.0:1787828385.639082:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787828385.639085:0:40701:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787828385.639088:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4297859840 00000100:00000001:2.0:1787828385.639088:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828385.639092:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787828385.639095:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828385.639097:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828385.639098:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1787828385.639100:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 00000008:00100000:2.0:1787828385.639100:0:40701:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000008:00000001:2.0:1787828385.639105:0:40701:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828385.639106:0:40701:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828385.639109:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4297859840 02000000:00000001:2.0:1787828385.639110:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787828385.639112:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828385.639115:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787828385.639116:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:0.0:1787828385.639117:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 02000000:00000001:2.0:1787828385.639117:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828385.639135:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787828385.639136:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828385.639142:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787828385.639144:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2d2c0 x1874670646724864/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787828385.639146:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00000001:2.0:1787828385.639153:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00100000:0.0:1787828385.639155:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828385/real 1787828385] req@ffff8d30c9e2c780 x1874670646724480/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828410 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787828385.639156:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828385.639162:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:0.0:1787828385.639171:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787828385.639173:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787828385.639175:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787828385.639178:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787828385.639179:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828385.639181:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828385.639182:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787828385.639186:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2c780 x1874670646724480/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828410 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787828385.639192:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828385.639194:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787828385.639196:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828385.639198:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828385.639198:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828385.639200:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787828385.639202:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787828385.639203:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828385.639206:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787828385.639208:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787828385.639210:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646724480:192.168.203.143@tcp:250:kworker.0 00000100:00000001:0.0:1787828385.639213:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1787828385.639214:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787828385.639215:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828385.639216:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787828385.639217:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787828385.639219:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828385.639220:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646724608:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787828385.639223:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787828385.639224:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787828385.639225:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828385.639226:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1787828385.639228:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828385.639231:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787828385.639232:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417875968 : -126234291675648 : ffff8d30c9e2de00) 00000100:00000001:0.0:1787828385.639236:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1787828385.639243:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:0.0:1787828385.639245:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1787828385.639247:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828385.639248:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828385.639249:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787828385.639251:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828385.639254:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4297858582 00000100:00000001:0.0:1787828385.639256:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4297859840 00000100:00000001:0.0:1787828385.639257:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4297858582 00000100:00000001:0.0:1787828385.639259:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4297859840 00000100:00000001:0.0:1787828385.639260:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787828385.639262:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828385.639263:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787828385.639265:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828385/real 1787828385] req@ffff8d30c9e2de00 x1874670646724608/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828410 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787828385.639270:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787828385.639271:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787828385.639272:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787828385.639273:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787828385.639273:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828385.639274:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828385.639275:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787828385.639277:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2de00 x1874670646724608/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828410 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787828385.639282:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828385.639283:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787828385.639285:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828385.639286:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828385.639287:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828385.639288:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787828385.639289:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787828385.639290:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828385.639292:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787828385.639293:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787828385.639295:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646724608:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787828385.639298:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828385.639299:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828385.639301:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787828385.639302:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787828385.639305:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787828385.639307:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787828385.639308:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787828385.639312:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787828385.639313:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787828385.639316:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787828385.639317:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828385.639318:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828385.639319:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787828385.639321:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787828385.639323:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787828385.639324:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787828385.639325:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787828385.639327:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787828385.639328:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787828385.639331:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787828385.639332:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828385.639334:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828385.639340:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787828385.639342:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828385.639343:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787828385.639345:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787828385.639346:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1787828385.639347:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787828385.639348:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828385.639349:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787828385.639351:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787828385.639352:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828385.639354:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646724736:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787828385.639356:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787828385.639358:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787828385.639358:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828385.639360:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1787828385.639361:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828385.639364:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787828385.639365:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417883648 : -126234291667968 : ffff8d30c9e2fc00) 00000100:00000001:0.0:1787828385.639368:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1787828385.639400:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:0.0:1787828385.639403:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1787828385.639405:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828385.639406:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828385.639407:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787828385.639409:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828385.639411:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4297858582 00000100:00000001:0.0:1787828385.639413:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4297859840 00000100:00000001:0.0:1787828385.639416:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4297858582 00000100:00000001:0.0:1787828385.639417:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4297859840 00000100:00000001:0.0:1787828385.639418:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787828385.639420:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828385.639421:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787828385.639424:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828385/real 1787828385] req@ffff8d30c9e2fc00 x1874670646724736/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828410 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787828385.639429:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787828385.639430:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787828385.639431:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787828385.639432:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787828385.639433:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828385.639435:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828385.639436:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787828385.639438:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2fc00 x1874670646724736/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828410 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787828385.639442:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828385.639443:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787828385.639445:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828385.639446:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828385.639446:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828385.639447:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787828385.639448:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787828385.639450:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828385.639452:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787828385.639453:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787828385.639456:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646724736:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787828385.639458:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1787828385.639459:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787828385.639460:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828385.639461:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787828385.639462:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787828385.639463:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828385.639465:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2d2c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646724864:192.168.203.143@tcp:8:kworker.0 00000100:00000001:0.0:1787828385.639467:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787828385.639468:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787828385.639469:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828385.639470:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1787828385.639472:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828385.639475:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787828385.639476:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417873088 : -126234291678528 : ffff8d30c9e2d2c0) 00000100:00000001:0.0:1787828385.639478:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1787828385.639483:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:0.0:1787828385.639486:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1787828385.639487:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828385.639488:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828385.639489:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787828385.639491:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828385.639493:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4297858595 00000100:00000001:0.0:1787828385.639495:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4297859840 00000100:00000001:0.0:1787828385.639496:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4297858595 00000100:00000001:0.0:1787828385.639498:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4297859840 00000100:00000001:0.0:1787828385.639499:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787828385.639500:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828385.639501:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787828385.639504:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828385/real 1787828385] req@ffff8d30c9e2d2c0 x1874670646724864/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828410 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787828385.639517:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787828385.639518:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787828385.639519:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787828385.639521:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787828385.639522:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828385.639523:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828385.639524:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787828385.639526:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2d2c0 x1874670646724864/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828410 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787828385.639530:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828385.639532:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787828385.639533:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828385.639534:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828385.639535:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828385.639536:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787828385.639538:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787828385.639539:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828385.639541:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787828385.639542:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787828385.639544:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2d2c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646724864:192.168.203.143@tcp:8:kworker.0 00000100:00000001:0.0:1787828385.639546:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828385.639547:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828385.639548:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787828385.639548:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787828385.639550:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787828385.639550:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787828385.639551:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787828385.639553:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787828385.639553:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787828385.639555:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787828385.639556:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828385.639557:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828385.639558:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787828385.639559:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787828385.639561:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787828385.639561:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787828385.639562:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787828385.639563:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787828385.639564:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787828385.639565:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787828385.639565:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828385.639566:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828385.639568:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787828385.639569:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828385.639575:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787828385.639576:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828386.534708:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828386.534714:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787828386.534779:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787828386.534784:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787828386.534788:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828387.558695:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828387.558701:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787828387.558733:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787828387.558738:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787828387.558741:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828388.582703:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828388.582707:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787828388.582768:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1787828388.582773:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1787828388.582776:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828389.606689:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828389.606695:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787828389.606755:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787828389.606760:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787828389.606763:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828390.630709:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828390.630716:0:815:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1787828390.630718:0:815:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1787828390.630721:0:815:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828390.630726:0:815:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1787828390.630728:0:815:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828390.630730:0:815:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828390.630733:0:815:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787828390.630801:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787828390.630807:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787828390.630811:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787828390.758697:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828390.758706:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1787828390.758708:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787828390.758711:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787828390.758715:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787828390.758721:0:40701:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 3191 00000100:00080000:2.0:1787828390.758730:0:40701:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787828390.758733:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828390.758734:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787828390.758738:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787828390.758741:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787828390.758743:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:2.0:1787828390.758745:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:2.0:1787828390.758747:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828390.758748:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:2.0:1787828390.758752:0:40701:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787828390.758755:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828390.758757:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787828390.758761:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828390.758763:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828390.758765:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:2.0:1787828390.758767:0:40701:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828390.758777:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787828390.758787:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828390.758790:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787828390.758792:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787828390.758796:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828390.758808:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787828390.758816:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2e1c0 x1874670646724992/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787828390.758825:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787828390.758829:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828390.758842:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787828390.758845:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828390.758848:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1787828390.758849:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787828390.758850:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787828390.758852:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787828390.758854:0:40701:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3191 00000100:00080000:2.0:1787828390.758856:0:40701:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787828390.758857:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828390.758858:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787828390.758860:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787828390.758861:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787828390.758863:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000020:00000001:2.0:1787828390.758865:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:2.0:1787828390.758867:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828390.758868:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:2.0:1787828390.758871:0:40701:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787828390.758874:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828390.758875:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787828390.758880:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828390.758882:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828390.758884:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1787828390.758888:0:40701:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00000001:0.0:1787828390.758890:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000008:00000001:2.0:1787828390.758891:0:40701:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828390.758896:0:40701:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828390.758898:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 02000000:00000001:2.0:1787828390.758900:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:0.0:1787828390.758904:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 02000000:00000001:2.0:1787828390.758904:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828390.758909:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:0.0:1787828390.758910:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 02000000:00000001:2.0:1787828390.758911:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:0.0:1787828390.758915:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828390.758915:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828390.758920:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828390.758923:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000100:00100000:2.0:1787828390.758927:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2d680 x1874670646725120/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:0.0:1787828390.758929:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787828390.758937:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828390.758941:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787828390.758944:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828390.758948:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646724992:192.168.203.143@tcp:250:kworker.0 00000100:00000001:2.0:1787828390.758948:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787828390.758954:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1787828390.758960:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00080000:2.0:1787828390.758961:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1787828390.758965:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 02000000:00000001:0.0:1787828390.758969:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000100:00080000:2.0:1787828390.758969:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 02000000:00000001:0.0:1787828390.758974:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828390.758975:0:40701:0:(import.c:485:import_select_connection()) Process entered 02000000:00000001:0.0:1787828390.758980:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00080000:2.0:1787828390.758984:0:40701:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3191 02000000:00000001:0.0:1787828390.758988:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787828390.758991:0:40701:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787828390.758996:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828390.758999:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787828390.759002:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787828390.759004:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:0.0:1787828390.759006:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000020:00000001:2.0:1787828390.759007:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:0.0:1787828390.759011:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417876928 : -126234291674688 : ffff8d30c9e2e1c0) 00000020:00000001:2.0:1787828390.759013:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:0.0:1787828390.759017:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:2.0:1787828390.759019:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828390.759023:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:2.0:1787828390.759027:0:40701:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787828390.759030:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828390.759031:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787828390.759034:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828390.759036:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828390.759038:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1787828390.759039:0:40701:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1787828390.759041:0:40701:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828390.759043:0:40701:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828390.759048:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787828390.759049:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828390.759051:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787828390.759052:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787828390.759053:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:0.0:1787828390.759059:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:2.0:1787828390.759060:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787828390.759067:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2cb40 x1874670646725248/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787828390.759073:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1787828390.759077:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787828390.759080:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828390.759082:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828390.759085:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828390.759088:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828390.759089:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00080000:2.0:1787828390.759095:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1787828390.759097:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787828390.759105:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1787828390.759106:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4297864960 00000100:00000001:2.0:1787828390.759108:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787828390.759113:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787828390.759116:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:0.0:1787828390.759120:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 00000100:00080000:2.0:1787828390.759121:0:40701:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3191 00000100:00080000:2.0:1787828390.759130:0:40701:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787828390.759131:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4297864960 00000100:00000001:2.0:1787828390.759134:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828390.759138:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787828390.759140:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:0.0:1787828390.759141:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 00000020:00000001:2.0:1787828390.759141:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787828390.759144:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000020:00000001:2.0:1787828390.759147:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:0.0:1787828390.759149:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787828390.759149:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828390.759152:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:0.0:1787828390.759156:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787828390.759157:0:40701:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787828390.759163:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828390.759164:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 02000000:00000001:2.0:1787828390.759165:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787828390.759170:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828390.759172:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828390.759175:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00100000:0.0:1787828390.759176:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828390/real 1787828390] req@ffff8d30c9e2e1c0 x1874670646724992/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828415 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:2.0:1787828390.759177:0:40701:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000008:00000001:2.0:1787828390.759182:0:40701:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828390.759187:0:40701:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:0.0:1787828390.759198:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 02000000:00000001:2.0:1787828390.759201:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000400:00000001:0.0:1787828390.759202:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 02000000:00000001:2.0:1787828390.759204:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828390.759207:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787828390.759209:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:0.0:1787828390.759210:0:2390:0:(events.c:78:reply_in_callback()) Process entered 02000000:00000001:2.0:1787828390.759211:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828390.759215:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787828390.759219:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828390.759220:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828390.759225:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00100000:2.0:1787828390.759226:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2d2c0 x1874670646725376/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787828390.759229:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828390.759238:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00100000:0.0:1787828390.759241:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2e1c0 x1874670646724992/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828415 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787828390.759241:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828390.759246:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828390.759258:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828390.759263:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787828390.759269:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828390.759271:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828390.759273:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828390.759276:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787828390.759278:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787828390.759282:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828390.759287:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787828390.759290:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787828390.759295:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646724992:192.168.203.143@tcp:250:kworker.0 00000100:00000001:0.0:1787828390.759301:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828390.759303:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828390.759305:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787828390.759308:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787828390.759312:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787828390.759315:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787828390.759317:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787828390.759321:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787828390.759323:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787828390.759327:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787828390.759329:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828390.759332:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828390.759355:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787828390.759357:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828390.759360:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787828390.759361:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787828390.759363:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1787828390.759365:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787828390.759366:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828390.759370:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787828390.759372:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787828390.759375:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828390.759378:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2d680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646725120:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787828390.759383:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787828390.759385:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787828390.759387:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828390.759390:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1787828390.759393:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828390.759400:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787828390.759402:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417874048 : -126234291677568 : ffff8d30c9e2d680) 00000100:00000001:0.0:1787828390.759407:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1787828390.759425:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:0.0:1787828390.759431:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1787828390.759446:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828390.759447:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828390.759455:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787828390.759459:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828390.759465:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4297863702 00000100:00000001:0.0:1787828390.759469:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4297864960 00000100:00000001:0.0:1787828390.759473:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4297863702 00000100:00000001:0.0:1787828390.759476:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4297864960 00000100:00000001:0.0:1787828390.759479:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787828390.759482:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828390.759485:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787828390.759492:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828390/real 1787828390] req@ffff8d30c9e2d680 x1874670646725120/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828415 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787828390.759504:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787828390.759506:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787828390.759510:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787828390.759512:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787828390.759514:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828390.759525:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828390.759527:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787828390.759533:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2d680 x1874670646725120/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828415 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787828390.759542:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828390.759545:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787828390.759548:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828390.759551:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828390.759553:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828390.759555:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787828390.759558:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787828390.759561:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828390.759566:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787828390.759569:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787828390.759574:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2d680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646725120:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787828390.759579:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1787828390.759580:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787828390.759582:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828390.759585:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787828390.759587:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787828390.759589:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828390.759593:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646725248:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787828390.759597:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787828390.759631:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787828390.759634:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828390.759636:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1787828390.759640:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828390.759647:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787828390.759649:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417871168 : -126234291680448 : ffff8d30c9e2cb40) 00000100:00000001:0.0:1787828390.759654:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1787828390.759670:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:0.0:1787828390.759676:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1787828390.759679:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828390.759681:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828390.759683:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787828390.759686:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828390.759690:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4297863703 00000100:00000001:0.0:1787828390.759694:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4297864960 00000100:00000001:0.0:1787828390.759696:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4297863703 00000100:00000001:0.0:1787828390.759699:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4297864960 00000100:00000001:0.0:1787828390.759701:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787828390.759704:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828390.759706:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787828390.759711:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828390/real 1787828390] req@ffff8d30c9e2cb40 x1874670646725248/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828415 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787828390.759730:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787828390.759732:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787828390.759735:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787828390.759737:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787828390.759739:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828390.759741:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828390.759743:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787828390.759748:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2cb40 x1874670646725248/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828415 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787828390.759757:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828390.759760:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787828390.759762:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828390.759764:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828390.759766:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828390.759768:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787828390.759770:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787828390.759773:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828390.759777:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787828390.759780:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787828390.759784:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646725248:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787828390.759789:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1787828390.759791:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787828390.759793:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828390.759795:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787828390.759797:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787828390.759800:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828390.759803:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2d2c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646725376:192.168.203.143@tcp:8:kworker.0 00000100:00000001:0.0:1787828390.759807:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787828390.759809:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787828390.759812:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828390.759813:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1787828390.759817:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828390.759823:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787828390.759825:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417873088 : -126234291678528 : ffff8d30c9e2d2c0) 00000100:00000001:0.0:1787828390.759829:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1787828390.759844:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:0.0:1787828390.759849:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1787828390.759852:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828390.759854:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828390.759856:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787828390.759859:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828390.759863:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4297863716 00000100:00000001:0.0:1787828390.759866:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4297864960 00000100:00000001:0.0:1787828390.759869:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4297863716 00000100:00000001:0.0:1787828390.759871:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4297864960 00000100:00000001:0.0:1787828390.759874:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787828390.759876:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828390.759879:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787828390.759884:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828390/real 1787828390] req@ffff8d30c9e2d2c0 x1874670646725376/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828415 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787828390.759894:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787828390.759895:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787828390.759898:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787828390.759900:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787828390.759902:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828390.759904:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828390.759906:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787828390.759911:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2d2c0 x1874670646725376/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828415 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787828390.759920:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828390.759922:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787828390.759925:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828390.759927:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828390.759929:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828390.759931:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787828390.759933:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787828390.759936:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828390.759940:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787828390.759943:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787828390.759947:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2d2c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646725376:192.168.203.143@tcp:8:kworker.0 00000100:00000001:0.0:1787828390.759951:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828390.759954:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828390.759955:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787828390.759957:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787828390.759960:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787828390.759962:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787828390.759963:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787828390.759966:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787828390.759968:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787828390.759972:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787828390.759973:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828390.759975:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828390.759977:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787828390.759978:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787828390.759980:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787828390.759981:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787828390.759983:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787828390.759985:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787828390.759986:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787828390.759989:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787828390.759990:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828390.759992:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828390.759994:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787828390.759995:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787828390.759997:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787828390.759999:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787828390.760000:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787828390.760002:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787828390.760004:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787828390.760006:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787828390.760008:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828390.760010:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828390.760015:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787828390.760017:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828390.760019:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787828390.760021:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828391.654659:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828391.654664:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787828391.654720:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1787828391.654723:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1787828391.654725:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828392.678763:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828392.678771:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787828392.678877:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787828392.678882:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787828392.678887:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828393.702753:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828393.702764:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787828393.702888:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787828393.702898:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787828393.702904:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828394.726716:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828394.726724:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787828394.726836:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1787828394.726842:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1787828394.726847:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828395.750663:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828395.750668:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787828395.750737:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787828395.750742:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787828395.750746:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787828395.878782:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828395.878793:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1787828395.878796:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787828395.878800:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787828395.878805:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787828395.878813:0:40701:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 3196 00000100:00080000:2.0:1787828395.878825:0:40701:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787828395.878829:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828395.878831:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787828395.878836:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787828395.878839:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787828395.878843:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:2.0:1787828395.878846:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:2.0:1787828395.878849:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828395.878851:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:2.0:1787828395.878856:0:40701:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787828395.878859:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828395.878863:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787828395.878867:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828395.878871:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828395.878874:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:2.0:1787828395.878877:0:40701:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828395.878887:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787828395.878890:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828395.878894:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787828395.878897:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787828395.878902:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828395.878929:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787828395.878941:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2fc00 x1874670646725504/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787828395.878954:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787828395.878960:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828395.878975:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787828395.878980:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828395.878985:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1787828395.878988:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787828395.878990:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787828395.878995:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787828395.878999:0:40701:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3196 00000100:00000001:0.0:1787828395.879003:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00080000:2.0:1787828395.879004:0:40701:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787828395.879008:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787828395.879010:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1787828395.879011:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:0.0:1787828395.879015:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000020:00000001:2.0:1787828395.879016:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787828395.879019:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:0.0:1787828395.879020:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000020:00000001:2.0:1787828395.879022:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:0.0:1787828395.879023:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1787828395.879026:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:2.0:1787828395.879029:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 02000000:00000001:0.0:1787828395.879030:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:2.0:1787828395.879031:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 02000000:00000001:0.0:1787828395.879034:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00080000:2.0:1787828395.879054:0:40701:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 02000000:00000001:0.0:1787828395.879056:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828395.879065:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828395.879067:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646725504:192.168.203.143@tcp:250:kworker.0 02000000:00000001:2.0:1787828395.879072:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:0.0:1787828395.879075:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787828395.879081:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1787828395.879083:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787828395.879085:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828395.879093:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1787828395.879093:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828395.879101:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828395.879112:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1787828395.879116:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787828395.879121:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417883648 : -126234291667968 : ffff8d30c9e2fc00) 00000008:00100000:2.0:1787828395.879121:0:40701:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00000001:0.0:1787828395.879129:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000008:00000001:2.0:1787828395.879132:0:40701:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828395.879136:0:40701:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828395.879158:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787828395.879160:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828395.879162:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787828395.879163:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787828395.879165:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828395.879170:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787828395.879175:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2de00 x1874670646725632/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787828395.879182:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000800:00000100:0.0:1787828395.879187:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:2.0:1787828395.879187:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828395.879194:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787828395.879197:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1787828395.879199:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00080000:2.0:1787828395.879201:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1787828395.879220:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:0.0:1787828395.879222:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00080000:2.0:1787828395.879223:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787828395.879227:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828395.879229:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:0.0:1787828395.879234:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00080000:2.0:1787828395.879237:0:40701:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3196 00000100:00000001:0.0:1787828395.879246:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787828395.879247:0:40701:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787828395.879254:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4297870080 00000100:00000001:2.0:1787828395.879262:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787828395.879264:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 00000100:00000001:2.0:1787828395.879286:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:0.0:1787828395.879292:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4297870080 00000020:00000001:2.0:1787828395.879293:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787828395.879325:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787828395.879330:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:0.0:1787828395.879331:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 00000020:00000001:2.0:1787828395.879353:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:0.0:1787828395.879355:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787828395.879358:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787828395.879359:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828395.879361:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:0.0:1787828395.879364:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00080000:2.0:1787828395.879367:0:40701:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787828395.879370:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828395.879373:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828395/real 1787828395] req@ffff8d30c9e2fc00 x1874670646725504/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828420 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1787828395.879373:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787828395.879378:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828395.879383:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828395.879387:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1787828395.879390:0:40701:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1787828395.879394:0:40701:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:0.0:1787828395.879395:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000001:2.0:1787828395.879397:0:40701:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:0.0:1787828395.879398:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 02000000:00000001:2.0:1787828395.879422:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:0.0:1787828395.879423:0:2390:0:(events.c:78:reply_in_callback()) Process entered 02000000:00000001:2.0:1787828395.879426:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828395.879428:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787828395.879430:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:0.0:1787828395.879433:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:2.0:1787828395.879434:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:0.0:1787828395.879440:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 02000000:00000001:2.0:1787828395.879440:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828395.879445:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828395.879450:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828395.879454:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2fc00 x1874670646725504/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828420 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787828395.879462:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2c780 x1874670646725760/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787828395.879489:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828395.879496:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787828395.879499:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787828395.879502:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828395.879505:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828395.879507:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828395.879511:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828395.879512:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:2.0:1787828395.879521:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1787828395.879522:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787828395.879526:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787828395.879529:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1787828395.879534:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787828395.879535:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828395.879539:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787828395.879545:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:2.0:1787828395.879545:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:0.0:1787828395.879553:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00080000:2.0:1787828395.879554:0:40701:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3196 00000100:00080000:2.0:1787828395.879562:0:40701:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1787828395.879580:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646725504:192.168.203.143@tcp:250:kworker.0 00000100:00000001:2.0:1787828395.879586:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828395.879590:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:0.0:1787828395.879592:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:2.0:1787828395.879594:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:0.0:1787828395.879597:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000020:00000001:2.0:1787828395.879598:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:0.0:1787828395.879603:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 00000020:00000001:2.0:1787828395.879604:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 02000000:00000001:0.0:1787828395.879609:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 00000020:00000001:2.0:1787828395.879609:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:2.0:1787828395.879615:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 02000000:00000001:0.0:1787828395.879616:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00000001:2.0:1787828395.879617:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787828395.879622:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787828395.879627:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 00000100:00080000:2.0:1787828395.879627:0:40701:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787828395.879648:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828395.879650:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787828395.879651:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787828395.879653:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 02000000:00000001:2.0:1787828395.879655:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787828395.879658:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 02000000:00000001:2.0:1787828395.879659:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828395.879662:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828395.879663:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1787828395.879667:0:40701:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00000001:0.0:1787828395.879668:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000008:00000001:2.0:1787828395.879674:0:40701:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828395.879676:0:40701:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828395.879716:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:0.0:1787828395.879717:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 02000000:00000001:2.0:1787828395.879718:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828395.879724:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828395.879724:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787828395.879727:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787828395.879731:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828395.879734:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787828395.879739:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828395.879740:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787828395.879746:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00100000:2.0:1787828395.879748:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2e940 x1874670646725888/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787828395.879751:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787828395.879758:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828395.879763:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 02000000:00000001:0.0:1787828395.879766:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:2.0:1787828395.879767:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828395.879791:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787828395.879795:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828395.879799:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828395.879811:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646725632:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787828395.879816:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787828395.879821:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787828395.879823:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828395.879825:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1787828395.879830:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828395.879839:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787828395.879841:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417875968 : -126234291675648 : ffff8d30c9e2de00) 00000100:00000001:0.0:1787828395.879847:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1787828395.879868:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:0.0:1787828395.879876:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1787828395.879881:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828395.879883:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828395.879886:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787828395.879890:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828395.879897:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4297868823 00000100:00000001:0.0:1787828395.879920:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4297870080 00000100:00000001:0.0:1787828395.879923:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4297868823 00000100:00000001:0.0:1787828395.879926:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4297870080 00000100:00000001:0.0:1787828395.879929:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787828395.879931:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828395.879933:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787828395.879939:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828395/real 1787828395] req@ffff8d30c9e2de00 x1874670646725632/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828420 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787828395.879949:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787828395.879950:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787828395.879952:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787828395.879954:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787828395.879955:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828395.879956:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828395.879957:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787828395.879961:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2de00 x1874670646725632/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828420 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787828395.879966:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828395.879968:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787828395.879970:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828395.879971:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828395.879972:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828395.879973:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787828395.879974:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787828395.879976:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828395.879978:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787828395.879980:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787828395.879982:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646725632:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787828395.879985:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1787828395.879986:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787828395.879987:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828395.879989:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787828395.879991:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787828395.879993:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828395.879995:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646725760:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787828395.879997:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787828395.879999:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787828395.880000:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828395.880001:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1787828395.880003:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828395.880007:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787828395.880008:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417870208 : -126234291681408 : ffff8d30c9e2c780) 00000100:00000001:0.0:1787828395.880011:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1787828395.880021:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:0.0:1787828395.880024:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1787828395.880026:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828395.880027:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828395.880028:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787828395.880030:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828395.880033:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4297868823 00000100:00000001:0.0:1787828395.880035:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4297870080 00000100:00000001:0.0:1787828395.880037:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4297868823 00000100:00000001:0.0:1787828395.880038:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4297870080 00000100:00000001:0.0:1787828395.880040:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787828395.880041:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828395.880042:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787828395.880045:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828395/real 1787828395] req@ffff8d30c9e2c780 x1874670646725760/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828420 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787828395.880050:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787828395.880051:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787828395.880052:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787828395.880054:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787828395.880055:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828395.880056:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828395.880057:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787828395.880060:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2c780 x1874670646725760/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828420 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787828395.880065:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828395.880066:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787828395.880068:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828395.880069:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828395.880070:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828395.880071:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787828395.880072:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787828395.880074:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828395.880076:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787828395.880077:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787828395.880079:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646725760:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787828395.880082:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828395.880083:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828395.880084:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787828395.880085:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787828395.880087:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787828395.880088:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787828395.880089:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787828395.880090:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787828395.880091:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787828395.880094:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787828395.880095:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828395.880096:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828395.880097:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787828395.880098:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787828395.880099:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787828395.880100:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787828395.880101:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787828395.880102:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787828395.880103:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787828395.880105:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787828395.880106:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828395.880107:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828395.880110:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787828395.880112:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828395.880113:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787828395.880114:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787828395.880115:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1787828395.880116:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787828395.880118:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828395.880119:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787828395.880120:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787828395.880122:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828395.880124:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646725888:192.168.203.143@tcp:8:kworker.0 00000100:00000001:0.0:1787828395.880126:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787828395.880127:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787828395.880128:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828395.880130:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1787828395.880132:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828395.880136:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787828395.880137:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417878848 : -126234291672768 : ffff8d30c9e2e940) 00000100:00000001:0.0:1787828395.880139:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1787828395.880146:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:0.0:1787828395.880149:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1787828395.880150:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828395.880167:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828395.880169:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787828395.880172:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828395.880175:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4297868836 00000100:00000001:0.0:1787828395.880178:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4297870080 00000100:00000001:0.0:1787828395.880180:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4297868836 00000100:00000001:0.0:1787828395.880182:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4297870080 00000100:00000001:0.0:1787828395.880184:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787828395.880186:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828395.880187:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787828395.880191:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828395/real 1787828395] req@ffff8d30c9e2e940 x1874670646725888/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828420 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787828395.880205:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787828395.880206:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787828395.880209:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787828395.880210:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787828395.880211:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828395.880213:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828395.880214:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787828395.880217:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2e940 x1874670646725888/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828420 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787828395.880223:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828395.880225:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787828395.880227:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828395.880228:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828395.880230:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828395.880232:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787828395.880233:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787828395.880236:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828395.880239:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787828395.880240:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787828395.880243:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646725888:192.168.203.143@tcp:8:kworker.0 00000100:00000001:0.0:1787828395.880246:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828395.880248:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828395.880249:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787828395.880250:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787828395.880251:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787828395.880252:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787828395.880253:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787828395.880255:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787828395.880256:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787828395.880258:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787828395.880259:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828395.880261:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828395.880263:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787828395.880265:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828395.880267:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787828395.880268:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828396.774679:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828396.774685:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787828396.774774:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787828396.774778:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787828396.774780:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828397.798718:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828397.798725:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787828397.798802:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1787828397.798805:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1787828397.798808:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828398.822674:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828398.822681:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787828398.822741:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787828398.822745:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787828398.822748:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828399.846696:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828399.846700:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787828399.846776:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787828399.846780:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787828399.846783:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828400.870734:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828400.870744:0:815:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1787828400.870747:0:815:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1787828400.870751:0:815:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828400.870757:0:815:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1787828400.870760:0:815:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828400.870763:0:815:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828400.870766:0:815:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787828400.870871:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1787828400.870876:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1787828400.870880:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787828400.998707:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828400.998717:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1787828400.998720:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1787828400.998724:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787828400.998729:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1787828400.998736:0:468:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 3201 00000100:00080000:1.0:1787828400.998746:0:468:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1787828400.998749:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787828400.998751:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:1.0:1787828400.998754:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:1.0:1787828400.998757:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:1.0:1787828400.998761:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:1.0:1787828400.998763:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:1.0:1787828400.998764:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787828400.998765:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:1.0:1787828400.998769:0:468:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:1.0:1787828400.998770:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828400.998773:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1787828400.998776:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787828400.998779:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828400.998781:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:1.0:1787828400.998783:0:468:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828400.998796:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1787828400.998797:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828400.998801:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1787828400.998803:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1787828400.998807:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828400.998821:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787828400.998830:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d13ecf00 x1874670646726016/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787828400.998839:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787828400.998844:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828400.998859:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1787828400.998862:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828400.998864:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1787828400.998865:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1787828400.998866:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787828400.998868:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1787828400.998870:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3201 00000100:00080000:1.0:1787828400.998872:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1787828400.998874:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787828400.998876:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:1.0:1787828400.998877:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:1.0:1787828400.998879:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:1.0:1787828400.998881:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000020:00000001:1.0:1787828400.998884:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:1.0:1787828400.998886:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787828400.998887:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:1.0:1787828400.998889:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:1.0:1787828400.998890:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828400.998892:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:0.0:1787828400.998896:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:1.0:1787828400.998898:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787828400.998902:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828400.998905:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1787828400.998907:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787828400.998914:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000008:00100000:1.0:1787828400.998914:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00000001:0.0:1787828400.998922:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000008:00000001:1.0:1787828400.998922:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828400.998927:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828400.998927:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828400.998946:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787828400.998947:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787828400.998950:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787828400.998950:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828400.998960:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787828400.998962:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828400.998964:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1787828400.998969:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828400.998975:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d13ecf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646726016:192.168.203.143@tcp:250:kworker.0 00000100:00000001:1.0:1787828400.998975:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828400.998985:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00100000:1.0:1787828400.998990:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d13eda40 x1874670646726144/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:0.0:1787828400.998991:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787828400.998996:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828400.999003:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1787828400.999014:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828400.999014:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787828400.999020:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828400.999028:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1787828400.999031:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828400.999036:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1787828400.999037:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787828400.999038:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:0.0:1787828400.999044:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839541341952 : -126234168209664 : ffff8d30d13ecf00) 00000100:00080000:1.0:1787828400.999044:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787828400.999050:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:0.0:1787828400.999054:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00080000:1.0:1787828400.999054:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3201 00000100:00080000:1.0:1787828400.999060:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1787828400.999064:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787828400.999065:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:1.0:1787828400.999068:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:1.0:1787828400.999069:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:1.0:1787828400.999070:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000020:00000001:1.0:1787828400.999074:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:1.0:1787828400.999076:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787828400.999077:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:1.0:1787828400.999080:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:1.0:1787828400.999081:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828400.999083:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1787828400.999086:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787828400.999087:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828400.999089:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:1.0:1787828400.999090:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1787828400.999092:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828400.999094:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:0.0:1787828400.999100:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 02000000:00000001:1.0:1787828400.999103:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1787828400.999106:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828400.999109:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1787828400.999110:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1787828400.999112:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828400.999128:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828400.999130:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00100000:1.0:1787828400.999137:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d13ee1c0 x1874670646726272/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787828400.999142:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787828400.999145:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787828400.999149:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828400.999151:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828400.999153:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828400.999164:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00080000:1.0:1787828400.999164:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828400.999174:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1787828400.999175:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1787828400.999177:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787828400.999180:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:0.0:1787828400.999183:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1787828400.999185:0:468:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3201 00000100:00080000:1.0:1787828400.999203:0:468:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787828400.999206:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4297875200 00000100:00000001:1.0:1787828400.999206:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787828400.999212:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:1.0:1787828400.999218:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:1.0:1787828400.999220:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:1.0:1787828400.999222:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000020:00000001:1.0:1787828400.999224:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:1.0:1787828400.999226:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787828400.999227:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:1.0:1787828400.999231:0:468:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787828400.999232:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 00000100:00000001:1.0:1787828400.999234:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828400.999240:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1787828400.999243:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787828400.999246:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828400.999248:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1787828400.999250:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4297875200 00000008:00100000:1.0:1787828400.999251:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000008:00000001:1.0:1787828400.999256:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828400.999260:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828400.999282:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1787828400.999284:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828400.999285:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1787828400.999286:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:0.0:1787828400.999288:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 02000000:00000001:1.0:1787828400.999289:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828400.999306:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828400.999313:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00100000:1.0:1787828400.999315:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d13ee580 x1874670646726400/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787828400.999319:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828400.999330:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00000001:1.0:1787828400.999338:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787828400.999343:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828400.999344:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828400/real 1787828400] req@ffff8d30d13ecf00 x1874670646726016/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828425 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787828400.999346:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:0.0:1787828400.999362:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787828400.999363:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787828400.999367:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787828400.999373:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787828400.999377:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828400.999379:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828400.999380:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787828400.999384:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d13ecf00 x1874670646726016/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828425 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787828400.999390:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828400.999392:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787828400.999394:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828400.999395:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828400.999396:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828400.999398:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787828400.999399:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787828400.999402:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828400.999405:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787828400.999406:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787828400.999409:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d13ecf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646726016:192.168.203.143@tcp:250:kworker.0 00000100:00000001:0.0:1787828400.999412:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828400.999414:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828400.999415:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787828400.999417:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787828400.999420:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787828400.999422:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787828400.999424:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787828400.999427:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787828400.999429:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787828400.999432:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787828400.999433:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828400.999435:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828400.999441:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787828400.999442:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828400.999444:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787828400.999445:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787828400.999446:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1787828400.999448:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787828400.999449:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828400.999450:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787828400.999452:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787828400.999454:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828400.999456:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d13eda40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646726144:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787828400.999459:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787828400.999460:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787828400.999462:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828400.999463:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1787828400.999465:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828400.999470:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787828400.999471:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839541344832 : -126234168206784 : ffff8d30d13eda40) 00000100:00000001:0.0:1787828400.999474:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1787828400.999485:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:0.0:1787828400.999488:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1787828400.999489:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828400.999490:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828400.999492:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787828400.999493:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828400.999497:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4297873942 00000100:00000001:0.0:1787828400.999499:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4297875200 00000100:00000001:0.0:1787828400.999500:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4297873942 00000100:00000001:0.0:1787828400.999502:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4297875200 00000100:00000001:0.0:1787828400.999504:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787828400.999505:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828400.999506:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787828400.999509:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828400/real 1787828400] req@ffff8d30d13eda40 x1874670646726144/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828425 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787828400.999515:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787828400.999516:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787828400.999517:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787828400.999518:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787828400.999519:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828400.999520:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828400.999521:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787828400.999524:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d13eda40 x1874670646726144/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828425 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787828400.999536:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828400.999538:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787828400.999540:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828400.999541:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828400.999542:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828400.999543:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787828400.999545:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787828400.999546:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828400.999549:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787828400.999550:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787828400.999552:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d13eda40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646726144:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787828400.999555:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1787828400.999557:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787828400.999558:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828400.999559:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787828400.999560:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787828400.999561:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828400.999564:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d13ee1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646726272:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787828400.999566:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787828400.999567:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787828400.999569:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828400.999570:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1787828400.999572:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828400.999575:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787828400.999576:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839541346752 : -126234168204864 : ffff8d30d13ee1c0) 00000100:00000001:0.0:1787828400.999594:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1787828400.999602:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:0.0:1787828400.999605:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1787828400.999607:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828400.999608:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828400.999609:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787828400.999611:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828400.999640:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4297873943 00000100:00000001:0.0:1787828400.999643:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4297875200 00000100:00000001:0.0:1787828400.999645:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4297873943 00000100:00000001:0.0:1787828400.999646:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4297875200 00000100:00000001:0.0:1787828400.999648:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787828400.999649:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828400.999650:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787828400.999654:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828400/real 1787828400] req@ffff8d30d13ee1c0 x1874670646726272/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828425 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787828400.999659:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787828400.999660:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787828400.999662:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787828400.999664:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787828400.999665:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828400.999666:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828400.999667:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787828400.999670:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d13ee1c0 x1874670646726272/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828425 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787828400.999679:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828400.999681:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787828400.999683:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828400.999684:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828400.999685:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828400.999686:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787828400.999687:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787828400.999689:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828400.999691:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787828400.999692:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787828400.999694:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d13ee1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646726272:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787828400.999697:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1787828400.999698:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787828400.999699:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828400.999700:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787828400.999702:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787828400.999703:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828400.999705:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d13ee580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646726400:192.168.203.143@tcp:8:kworker.0 00000100:00000001:0.0:1787828400.999707:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787828400.999709:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787828400.999710:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828400.999712:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1787828400.999714:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828400.999718:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787828400.999719:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839541347712 : -126234168203904 : ffff8d30d13ee580) 00000100:00000001:0.0:1787828400.999722:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1787828400.999730:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:0.0:1787828400.999732:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1787828400.999734:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828400.999735:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828400.999736:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787828400.999738:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828400.999741:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4297873956 00000100:00000001:0.0:1787828400.999743:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4297875200 00000100:00000001:0.0:1787828400.999745:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4297873956 00000100:00000001:0.0:1787828400.999746:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4297875200 00000100:00000001:0.0:1787828400.999748:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787828400.999749:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828400.999750:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787828400.999753:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828400/real 1787828400] req@ffff8d30d13ee580 x1874670646726400/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828425 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787828400.999758:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787828400.999758:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787828400.999760:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787828400.999761:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787828400.999762:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828400.999763:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828400.999764:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787828400.999767:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d13ee580 x1874670646726400/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828425 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787828400.999771:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828400.999772:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787828400.999774:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828400.999775:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828400.999776:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828400.999777:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787828400.999778:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787828400.999779:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828400.999781:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787828400.999783:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787828400.999785:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d13ee580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646726400:192.168.203.143@tcp:8:kworker.0 00000100:00000001:0.0:1787828400.999788:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828400.999790:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828400.999791:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787828400.999792:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787828400.999793:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787828400.999794:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787828400.999795:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787828400.999797:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787828400.999798:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787828400.999799:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787828400.999800:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828400.999801:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828400.999802:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787828400.999803:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787828400.999804:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787828400.999805:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787828400.999806:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787828400.999807:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787828400.999808:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787828400.999817:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787828400.999818:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828400.999819:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828400.999820:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787828400.999821:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787828400.999822:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787828400.999823:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787828400.999824:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787828400.999826:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787828400.999827:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787828400.999828:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787828400.999829:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828400.999830:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828400.999832:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787828400.999833:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828400.999834:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787828400.999835:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828401.894738:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828401.894746:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787828401.894806:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787828401.894814:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787828401.894819:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828402.918686:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828402.918699:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787828402.918772:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787828402.918776:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787828402.918780:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828403.942716:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828403.942723:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787828403.942847:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1787828403.942853:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1787828403.942857:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828404.966696:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828404.966703:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787828404.966783:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787828404.966786:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787828404.966789:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828405.990709:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828405.990727:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787828405.990800:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787828405.990815:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787828405.990819:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000001:0.0:1787828406.118980:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787828406.118987:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787828406.118992:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1787828406.118996:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787828406.119000:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828406.119007:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787828406.119013:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787828406.119018:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828406.119024:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c70143c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646726528:192.168.203.143@tcp:250:kworker.0 00000100:00000001:0.0:1787828406.119031:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787828406.119036:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787828406.119041:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828406.119055:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1787828406.119062:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828406.119079:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787828406.119084:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369536448 : -126234340015168 : ffff8d30c70143c0) 00000100:00000001:0.0:1787828406.119092:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1787828406.119138:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:0.0:1787828406.119152:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1787828406.119157:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828406.119161:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828406.119168:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787828406.119176:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828406.119185:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4297880320 00000100:00000001:0.0:1787828406.119198:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1787828406.119209:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4297880320 00000100:00000001:0.0:1787828406.119216:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1787828406.119223:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787828406.119248:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828406.119255:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787828406.119267:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828406/real 1787828406] req@ffff8d30c70143c0 x1874670646726528/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828431 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787828406.119303:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787828406.119308:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787828406.119315:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787828406.119320:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787828406.119324:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828406.119328:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828406.119332:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787828406.119343:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c70143c0 x1874670646726528/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828431 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787828406.119364:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828406.119369:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787828406.119371:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828406.119372:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828406.119374:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828406.119377:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787828406.119379:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787828406.119381:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828406.119385:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787828406.119386:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787828406.119390:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c70143c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646726528:192.168.203.143@tcp:250:kworker.0 00000100:00000001:0.0:1787828406.119394:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828406.119395:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828406.119396:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787828406.119398:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787828406.119401:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787828406.119403:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787828406.119405:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787828406.119408:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787828406.119410:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787828406.119412:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787828406.119413:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828406.119415:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828406.119422:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787828406.119423:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828406.119425:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787828406.119437:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787828406.119439:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1787828406.119440:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787828406.119441:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828406.119443:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787828406.119445:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787828406.119447:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828406.119449:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c7014f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646726656:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787828406.119452:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787828406.119453:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787828406.119455:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828406.119456:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1787828406.119459:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828406.119463:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787828406.119464:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369539328 : -126234340012288 : ffff8d30c7014f00) 00000100:00000001:0.0:1787828406.119468:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1787828406.119478:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:0.0:1787828406.119482:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1787828406.119484:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828406.119485:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828406.119486:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787828406.119489:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828406.119504:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4297879062 00000100:00000001:0.0:1787828406.119506:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4297880320 00000100:00000001:0.0:1787828406.119508:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4297879062 00000100:00000001:0.0:1787828406.119510:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4297880320 00000100:00000001:0.0:1787828406.119512:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787828406.119513:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828406.119515:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787828406.119518:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828406/real 1787828406] req@ffff8d30c7014f00 x1874670646726656/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828431 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787828406.119524:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787828406.119525:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787828406.119527:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787828406.119528:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787828406.119529:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828406.119531:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828406.119532:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787828406.119535:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c7014f00 x1874670646726656/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828431 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787828406.119540:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828406.119542:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787828406.119544:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828406.119545:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828406.119546:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828406.119547:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787828406.119548:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787828406.119550:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828406.119553:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787828406.119554:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787828406.119557:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c7014f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646726656:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787828406.119559:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1787828406.119560:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787828406.119562:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828406.119563:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787828406.119564:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787828406.119566:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828406.119568:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c70152c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646726784:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787828406.119571:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787828406.119572:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787828406.119573:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828406.119574:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1787828406.119577:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828406.119580:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787828406.119581:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369540288 : -126234340011328 : ffff8d30c70152c0) 00000100:00000001:0.0:1787828406.119584:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1787828406.119591:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:0.0:1787828406.119594:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1787828406.119596:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828406.119597:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828406.119598:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787828406.119638:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828406.119641:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4297879063 00000100:00000001:0.0:1787828406.119644:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4297880320 00000100:00000001:0.0:1787828406.119646:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4297879063 00000100:00000001:0.0:1787828406.119648:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4297880320 00000100:00000001:0.0:1787828406.119649:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787828406.119651:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828406.119652:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787828406.119656:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828406/real 1787828406] req@ffff8d30c70152c0 x1874670646726784/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828431 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787828406.119662:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787828406.119663:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787828406.119665:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787828406.119666:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787828406.119668:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828406.119669:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828406.119670:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787828406.119673:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c70152c0 x1874670646726784/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828431 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787828406.119678:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828406.119680:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787828406.119682:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828406.119683:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828406.119684:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828406.119685:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787828406.119686:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787828406.119688:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828406.119691:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787828406.119693:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787828406.119695:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c70152c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646726784:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787828406.119698:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1787828406.119699:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787828406.119700:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828406.119702:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787828406.119703:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787828406.119705:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828406.119707:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c7014b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646726912:192.168.203.143@tcp:8:kworker.0 00000100:00000001:0.0:1787828406.119709:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787828406.119711:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787828406.119712:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828406.119713:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1787828406.119716:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828406.119720:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787828406.119721:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369538368 : -126234340013248 : ffff8d30c7014b40) 00000100:00000001:0.0:1787828406.119723:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1787828406.119732:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:0.0:1787828406.119735:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1787828406.119737:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828406.119738:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828406.119740:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787828406.119742:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828406.119744:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4297879076 00000100:00000001:0.0:1787828406.119746:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4297880320 00000100:00000001:0.0:1787828406.119748:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4297879076 00000100:00000001:0.0:1787828406.119750:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4297880320 00000100:00000001:0.0:1787828406.119751:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787828406.119753:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828406.119754:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787828406.119757:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828406/real 1787828406] req@ffff8d30c7014b40 x1874670646726912/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828431 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787828406.119763:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787828406.119765:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787828406.119766:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787828406.119768:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787828406.119769:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828406.119770:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828406.119771:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787828406.119774:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c7014b40 x1874670646726912/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828431 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787828406.119779:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828406.119781:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787828406.119782:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828406.119783:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828406.119784:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828406.119786:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787828406.119787:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787828406.119789:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828406.119791:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787828406.119792:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787828406.119794:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c7014b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646726912:192.168.203.143@tcp:8:kworker.0 00000100:00000001:0.0:1787828406.119797:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828406.119799:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828406.119799:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787828406.119801:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787828406.119802:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787828406.119803:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787828406.119804:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787828406.119806:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787828406.119806:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787828406.119808:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787828406.119809:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828406.119811:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828406.119812:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787828406.119813:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787828406.119814:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787828406.119815:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787828406.119816:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787828406.119828:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787828406.119828:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787828406.119829:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787828406.119830:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828406.119832:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828406.119832:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787828406.119833:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787828406.119834:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787828406.119835:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787828406.119836:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787828406.119837:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787828406.119838:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787828406.119839:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787828406.119840:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828406.119841:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828406.119843:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787828406.119845:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828406.119846:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787828406.119847:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828407.014679:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828407.014685:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787828407.014726:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1787828407.014729:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1787828407.014737:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828408.038702:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828408.038707:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787828408.038761:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787828408.038765:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787828408.038768:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828409.062776:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828409.062803:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787828409.062956:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787828409.062965:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787828409.062973:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828410.086757:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828410.086764:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787828410.086885:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1787828410.086889:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1787828410.086893:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828411.110679:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828411.110686:0:815:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1787828411.110687:0:815:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1787828411.110690:0:815:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828411.110693:0:815:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1787828411.110695:0:815:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828411.110696:0:815:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828411.110698:0:815:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787828411.110752:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787828411.110756:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787828411.110758:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787828411.238687:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787828411.238694:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787828411.238702:0:468:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 3211 00000100:00080000:3.0:1787828411.238728:0:468:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787828411.238730:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787828411.238732:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:3.0:1787828411.238736:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1787828411.238738:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1787828411.238740:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:3.0:1787828411.238742:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:3.0:1787828411.238744:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787828411.238744:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:3.0:1787828411.238748:0:468:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:3.0:1787828411.238750:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787828411.238752:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787828411.238754:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787828411.238756:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828411.238758:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:3.0:1787828411.238760:0:468:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787828411.238767:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787828411.238768:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828411.238771:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787828411.238790:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787828411.238794:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828411.238802:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787828411.238810:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c7016d00 x1874670646727040/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787828411.238819:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787828411.238823:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828411.238839:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787828411.238842:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828411.238846:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1787828411.238848:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787828411.238850:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787828411.238854:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787828411.238856:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3211 00000100:00080000:3.0:1787828411.238861:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787828411.238864:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787828411.238866:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:3.0:1787828411.238870:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1787828411.238872:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1787828411.238874:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000020:00000001:3.0:1787828411.238877:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:0.0:1787828411.238879:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1787828411.238880:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787828411.238884:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:0.0:1787828411.238886:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00080000:3.0:1787828411.238889:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787828411.238890:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1787828411.238894:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787828411.238895:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828411.238897:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787828411.238898:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787828411.238904:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787828411.238906:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787828411.238907:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787828411.238911:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828411.238912:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828411.238915:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00100000:0.0:1787828411.238919:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c7016d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646727040:192.168.203.143@tcp:250:kworker.0 00000008:00100000:3.0:1787828411.238920:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1787828411.238928:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828411.238929:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:3.0:1787828411.238930:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828411.238935:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1787828411.238936:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787828411.238940:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828411.238941:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828411.238941:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787828411.238944:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787828411.238947:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1787828411.238948:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828411.238955:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828411.238956:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787828411.238965:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c70170c0 x1874670646727168/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787828411.238971:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787828411.238973:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828411.238975:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787828411.238977:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828411.238979:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369547008 : -126234340004608 : ffff8d30c7016d00) 00000100:00080000:3.0:1787828411.238981:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1787828411.238987:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00080000:3.0:1787828411.238987:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1787828411.238989:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787828411.238991:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787828411.238994:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787828411.238995:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3211 00000100:00080000:3.0:1787828411.238998:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787828411.239000:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787828411.239001:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:3.0:1787828411.239003:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1787828411.239004:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1787828411.239006:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000020:00000001:3.0:1787828411.239007:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:3.0:1787828411.239009:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787828411.239010:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:3.0:1787828411.239013:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:3.0:1787828411.239015:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787828411.239016:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787828411.239018:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787828411.239019:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828411.239021:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:3.0:1787828411.239023:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1787828411.239026:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000008:00000001:3.0:1787828411.239027:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828411.239031:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787828411.239036:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787828411.239037:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828411.239039:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787828411.239039:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787828411.239040:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787828411.239044:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828411.239048:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787828411.239049:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828411.239054:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787828411.239057:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c7017840 x1874670646727296/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787828411.239059:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787828411.239067:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828411.239074:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787828411.239075:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4297885440 00000100:00000001:3.0:1787828411.239077:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828411.239080:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787828411.239082:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828411.239084:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1787828411.239086:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 00000100:00000001:3.0:1787828411.239086:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787828411.239089:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787828411.239092:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:0.0:1787828411.239094:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4297885440 00000100:00080000:3.0:1787828411.239095:0:468:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3211 00000100:00000001:0.0:1787828411.239101:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 00000100:00080000:3.0:1787828411.239102:0:468:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787828411.239106:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787828411.239107:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787828411.239108:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:0.0:1787828411.239112:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828411.239113:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:0.0:1787828411.239117:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000020:00000001:3.0:1787828411.239117:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1787828411.239121:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000020:00000001:3.0:1787828411.239123:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00100000:0.0:1787828411.239127:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828411/real 1787828411] req@ffff8d30c7016d00 x1874670646727040/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828436 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787828411.239127:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787828411.239130:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:3.0:1787828411.239139:0:468:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:3.0:1787828411.239146:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:0.0:1787828411.239148:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 02000000:00000001:3.0:1787828411.239148:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000400:00000001:0.0:1787828411.239152:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 02000000:00000001:3.0:1787828411.239154:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787828411.239156:0:2390:0:(events.c:78:reply_in_callback()) Process entered 02000000:00000001:3.0:1787828411.239157:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828411.239161:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1787828411.239162:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000008:00100000:3.0:1787828411.239164:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00000001:0.0:1787828411.239165:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000008:00000001:3.0:1787828411.239168:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828411.239169:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787828411.239172:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828411.239173:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:3.0:1787828411.239178:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00100000:0.0:1787828411.239180:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c7016d00 x1874670646727040/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828436 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1787828411.239181:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828411.239186:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787828411.239189:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787828411.239207:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828411.239208:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828411.239213:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787828411.239214:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828411.239216:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828411.239219:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828411.239221:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787828411.239222:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c7014000 x1874670646727424/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787828411.239223:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787828411.239226:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787828411.239249:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00000001:3.0:1787828411.239254:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00080000:0.0:1787828411.239256:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787828411.239258:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828411.239260:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000001:3.0:1787828411.239262:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828411.239265:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c7016d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646727040:192.168.203.143@tcp:250:kworker.0 00000100:00000001:0.0:1787828411.239272:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828411.239274:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828411.239275:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787828411.239277:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787828411.239280:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787828411.239282:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787828411.239283:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787828411.239287:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787828411.239288:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787828411.239291:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787828411.239292:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828411.239295:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828411.239302:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787828411.239304:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828411.239306:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787828411.239308:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787828411.239309:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1787828411.239311:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787828411.239312:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828411.239314:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787828411.239317:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787828411.239319:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828411.239323:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c70170c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646727168:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787828411.239326:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787828411.239329:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787828411.239331:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828411.239333:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1787828411.239336:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828411.239342:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787828411.239344:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369547968 : -126234340003648 : ffff8d30c70170c0) 00000100:00000001:0.0:1787828411.239348:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1787828411.239361:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:0.0:1787828411.239365:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1787828411.239367:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828411.239368:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828411.239369:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787828411.239371:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828411.239374:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4297884182 00000100:00000001:0.0:1787828411.239376:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4297885440 00000100:00000001:0.0:1787828411.239378:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4297884182 00000100:00000001:0.0:1787828411.239380:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4297885440 00000100:00000001:0.0:1787828411.239381:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787828411.239382:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828411.239384:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787828411.239387:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828411/real 1787828411] req@ffff8d30c70170c0 x1874670646727168/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828436 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787828411.239392:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787828411.239393:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787828411.239394:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787828411.239396:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787828411.239397:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828411.239398:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828411.239399:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787828411.239419:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c70170c0 x1874670646727168/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828436 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787828411.239423:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828411.239425:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787828411.239427:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828411.239429:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828411.239429:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828411.239431:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787828411.239432:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787828411.239434:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828411.239436:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787828411.239437:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787828411.239439:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c70170c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646727168:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787828411.239442:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1787828411.239443:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787828411.239444:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828411.239445:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787828411.239446:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787828411.239448:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828411.239450:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c7017840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646727296:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787828411.239452:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787828411.239453:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787828411.239454:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828411.239456:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1787828411.239458:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828411.239461:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787828411.239462:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369549888 : -126234340001728 : ffff8d30c7017840) 00000100:00000001:0.0:1787828411.239464:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1787828411.239472:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:0.0:1787828411.239475:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1787828411.239477:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828411.239478:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828411.239479:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787828411.239481:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828411.239483:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4297884182 00000100:00000001:0.0:1787828411.239484:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4297885440 00000100:00000001:0.0:1787828411.239486:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4297884182 00000100:00000001:0.0:1787828411.239488:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4297885440 00000100:00000001:0.0:1787828411.239489:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787828411.239491:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828411.239492:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787828411.239494:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828411/real 1787828411] req@ffff8d30c7017840 x1874670646727296/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828436 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787828411.239499:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787828411.239500:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787828411.239501:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787828411.239502:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787828411.239503:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828411.239504:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828411.239505:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787828411.239507:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c7017840 x1874670646727296/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828436 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787828411.239512:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828411.239513:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787828411.239515:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828411.239516:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828411.239517:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828411.239518:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787828411.239519:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787828411.239520:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828411.239522:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787828411.239524:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787828411.239526:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c7017840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646727296:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787828411.239528:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1787828411.239529:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787828411.239530:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828411.239531:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787828411.239532:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787828411.239534:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828411.239536:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c7014000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646727424:192.168.203.143@tcp:8:kworker.0 00000100:00000001:0.0:1787828411.239537:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787828411.239539:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787828411.239539:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828411.239540:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1787828411.239542:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828411.239545:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787828411.239546:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369535488 : -126234340016128 : ffff8d30c7014000) 00000100:00000001:0.0:1787828411.239548:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1787828411.239553:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:0.0:1787828411.239555:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1787828411.239557:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828411.239558:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828411.239559:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787828411.239560:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828411.239562:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4297884195 00000100:00000001:0.0:1787828411.239564:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4297885440 00000100:00000001:0.0:1787828411.239565:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4297884195 00000100:00000001:0.0:1787828411.239567:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4297885440 00000100:00000001:0.0:1787828411.239568:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787828411.239570:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828411.239571:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787828411.239573:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828411/real 1787828411] req@ffff8d30c7014000 x1874670646727424/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828436 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787828411.239578:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787828411.239579:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787828411.239580:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787828411.239581:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787828411.239581:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828411.239583:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828411.239584:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787828411.239586:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c7014000 x1874670646727424/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828436 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787828411.239592:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828411.239593:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787828411.239594:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828411.239595:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828411.239596:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828411.239597:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787828411.239598:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787828411.239600:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828411.239601:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787828411.239603:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787828411.239605:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c7014000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646727424:192.168.203.143@tcp:8:kworker.0 00000100:00000001:0.0:1787828411.239607:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828411.239608:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828411.239609:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787828411.239610:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787828411.239611:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787828411.239612:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787828411.239613:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787828411.239615:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787828411.239615:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787828411.239617:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787828411.239618:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828411.239619:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828411.239620:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787828411.239621:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787828411.239622:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787828411.239623:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787828411.239623:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787828411.239624:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787828411.239625:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787828411.239626:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787828411.239627:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828411.239628:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828411.239629:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787828411.239630:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787828411.239654:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787828411.239655:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787828411.239656:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787828411.239657:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787828411.239658:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787828411.239659:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787828411.239660:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828411.239662:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828411.239664:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787828411.239665:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828411.239666:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787828411.239668:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828412.134681:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828412.134689:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787828412.134829:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787828412.134836:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787828412.134856:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828413.158759:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828413.158769:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787828413.158877:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1787828413.158883:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1787828413.158906:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828414.182754:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828414.182762:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787828414.182872:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787828414.182878:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787828414.182883:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828415.206683:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828415.206689:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787828415.206757:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787828415.206760:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787828415.206764:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828416.230704:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828416.230711:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787828416.230788:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1787828416.230797:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1787828416.230801:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787828416.358782:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828416.358794:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1787828416.358798:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787828416.358802:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787828416.358809:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787828416.358819:0:468:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 3216 00000100:00080000:3.0:1787828416.358835:0:468:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787828416.358838:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787828416.358841:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:3.0:1787828416.358846:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1787828416.358849:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1787828416.358854:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:3.0:1787828416.358856:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:3.0:1787828416.358859:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787828416.358860:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:3.0:1787828416.358866:0:468:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:3.0:1787828416.358869:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787828416.358872:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787828416.358876:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787828416.358879:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828416.358882:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:3.0:1787828416.358885:0:468:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787828416.358903:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787828416.358906:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828416.358910:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787828416.358914:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787828416.358920:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828416.358936:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787828416.358948:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c7017c00 x1874670646727552/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787828416.358960:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787828416.358967:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828416.358994:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787828416.359000:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828416.359006:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1787828416.359008:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787828416.359011:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787828416.359016:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787828416.359023:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3216 00000100:00080000:3.0:1787828416.359030:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787828416.359034:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787828416.359038:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:3.0:1787828416.359043:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1787828416.359045:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1787828416.359049:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:0.0:1787828416.359052:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:3.0:1787828416.359054:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:0.0:1787828416.359063:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787828416.359064:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787828416.359068:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787828416.359070:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:0.0:1787828416.359073:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787828416.359077:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828416.359088:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000100:00080000:3.0:1787828416.359090:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 02000000:00000001:0.0:1787828416.359093:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787828416.359109:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828416.359110:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787828416.359123:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00100000:0.0:1787828416.359130:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c7017c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646727552:192.168.203.143@tcp:250:kworker.0 02000000:00000001:3.0:1787828416.359132:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787828416.359152:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1787828416.359153:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828416.359176:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:3.0:1787828416.359179:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 02000000:00000001:0.0:1787828416.359184:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828416.359190:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000008:00100000:3.0:1787828416.359192:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 02000000:00000001:0.0:1787828416.359202:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:3.0:1787828416.359203:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828416.359211:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787828416.359224:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:0.0:1787828416.359227:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 02000000:00000001:3.0:1787828416.359229:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828416.359231:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369550848 : -126234340000768 : ffff8d30c7017c00) 00000100:00000001:3.0:1787828416.359240:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:0.0:1787828416.359242:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 02000000:00000001:3.0:1787828416.359248:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787828416.359254:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828416.359268:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787828416.359279:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c7016940 x1874670646727680/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787828416.359301:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:3.0:1787828416.359316:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787828416.359321:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787828416.359322:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828416.359328:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828416.359335:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828416.359336:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828416.359339:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00080000:3.0:1787828416.359350:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1787828416.359351:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828416.359362:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4297890560 00000100:00080000:3.0:1787828416.359368:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1787828416.359374:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:0.0:1787828416.359378:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 00000100:00080000:3.0:1787828416.359379:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787828416.359387:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:0.0:1787828416.359391:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4297890560 00000100:00080000:3.0:1787828416.359399:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3216 00000100:00000001:0.0:1787828416.359400:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1787828416.359409:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787828416.359414:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787828416.359418:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787828416.359419:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00000001:3.0:1787828416.359431:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787828416.359434:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00100000:0.0:1787828416.359437:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828416/real 1787828416] req@ffff8d30c7017c00 x1874670646727552/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828441 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:3.0:1787828416.359449:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1787828416.359455:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000400:00000001:0.0:1787828416.359461:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000020:00000001:3.0:1787828416.359461:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000400:00000001:0.0:1787828416.359468:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000020:00000001:3.0:1787828416.359470:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:0.0:1787828416.359473:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787828416.359476:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787828416.359479:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787828416.359480:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:0.0:1787828416.359489:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:3.0:1787828416.359497:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787828416.359498:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828416.359505:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787828416.359507:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787828416.359514:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00100000:0.0:1787828416.359523:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c7017c00 x1874670646727552/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828441 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1787828416.359525:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787828416.359541:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828416.359551:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:3.0:1787828416.359562:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00000001:0.0:1787828416.359563:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828416.359576:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000008:00000001:3.0:1787828416.359576:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828416.359585:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787828416.359592:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787828416.359594:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828416.359600:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:0.0:1787828416.359639:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:3.0:1787828416.359643:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:0.0:1787828416.359645:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 02000000:00000001:3.0:1787828416.359648:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828416.359649:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828416.359661:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787828416.359663:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787828416.359666:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787828416.359670:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:3.0:1787828416.359677:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c7014000 x1874670646727808/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828416.359680:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787828416.359695:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000001:3.0:1787828416.359710:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00100000:0.0:1787828416.359718:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c7017c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646727552:192.168.203.143@tcp:250:kworker.0 00000100:00000001:3.0:1787828416.359719:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828416.359736:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828416.359738:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828416.359753:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00080000:3.0:1787828416.359754:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1787828416.359759:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 00000100:00080000:3.0:1787828416.359761:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 02000000:00000001:0.0:1787828416.359766:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 00000100:00000001:3.0:1787828416.359766:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787828416.359777:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 02000000:00000001:0.0:1787828416.359781:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00000001:3.0:1787828416.359783:0:468:0:(import.c:485:import_select_connection()) Process entered 00000020:00000001:0.0:1787828416.359790:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000100:00080000:3.0:1787828416.359791:0:468:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3216 00000020:00000001:0.0:1787828416.359800:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 00000100:00080000:3.0:1787828416.359802:0:468:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787828416.359807:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787828416.359809:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 02000000:00000001:0.0:1787828416.359812:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 00000020:00000001:3.0:1787828416.359813:0:468:0:(genops.c:792:class_conn2export()) Process entered 02000000:00000001:0.0:1787828416.359823:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000020:00000001:3.0:1787828416.359823:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1787828416.359832:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:0.0:1787828416.359836:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000020:00000001:3.0:1787828416.359837:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:3.0:1787828416.359848:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787828416.359849:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787828416.359855:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:0.0:1787828416.359857:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:3.0:1787828416.359876:0:468:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787828416.359880:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787828416.359880:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828416.359886:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787828416.359888:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:0.0:1787828416.359901:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:3.0:1787828416.359901:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787828416.359914:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828416.359915:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787828416.359926:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787828416.359926:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1787828416.359941:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000008:00100000:3.0:1787828416.359941:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00000001:0.0:1787828416.359945:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:3.0:1787828416.359951:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828416.359953:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:3.0:1787828416.359955:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828416.359959:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787828416.359963:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787828416.359965:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787828416.359970:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828416.359976:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c7016940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646727680:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787828416.359980:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:0.0:1787828416.359983:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1787828416.359987:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787828416.359988:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787828416.359995:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828416.359998:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1787828416.359998:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828416.360005:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828416.360014:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828416.360016:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787828416.360018:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369546048 : -126234340005568 : ffff8d30c7016940) 00000100:00000001:0.0:1787828416.360029:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00100000:3.0:1787828416.360038:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c7017840 x1874670646727936/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787828416.360050:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000800:00000100:0.0:1787828416.360060:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:3.0:1787828416.360061:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828416.360084:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828416.360087:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1787828416.360098:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828416.360103:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828416.360106:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787828416.360111:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828416.360117:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4297889303 00000100:00000001:0.0:1787828416.360121:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4297890560 00000100:00000001:0.0:1787828416.360124:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4297889303 00000100:00000001:0.0:1787828416.360127:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4297890560 00000100:00000001:0.0:1787828416.360129:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787828416.360132:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828416.360134:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787828416.360140:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828416/real 1787828416] req@ffff8d30c7016940 x1874670646727680/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828441 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787828416.360150:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787828416.360151:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787828416.360153:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787828416.360177:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787828416.360180:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828416.360182:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828416.360183:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787828416.360188:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c7016940 x1874670646727680/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828441 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787828416.360214:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828416.360227:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787828416.360231:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828416.360232:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828416.360234:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828416.360236:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787828416.360238:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787828416.360242:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828416.360246:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787828416.360248:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787828416.360253:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c7016940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646727680:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787828416.360258:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1787828416.360260:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787828416.360262:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828416.360265:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787828416.360268:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787828416.360271:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828416.360274:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c7014000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646727808:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787828416.360278:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787828416.360281:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787828416.360284:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828416.360286:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1787828416.360290:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828416.360296:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787828416.360298:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369535488 : -126234340016128 : ffff8d30c7014000) 00000100:00000001:0.0:1787828416.360303:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1787828416.360319:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:0.0:1787828416.360325:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1787828416.360327:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828416.360329:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828416.360331:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787828416.360335:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828416.360339:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4297889303 00000100:00000001:0.0:1787828416.360342:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4297890560 00000100:00000001:0.0:1787828416.360344:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4297889303 00000100:00000001:0.0:1787828416.360347:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4297890560 00000100:00000001:0.0:1787828416.360349:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787828416.360352:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828416.360354:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787828416.360358:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828416/real 1787828416] req@ffff8d30c7014000 x1874670646727808/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828441 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787828416.360376:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787828416.360377:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787828416.360380:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787828416.360382:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787828416.360384:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828416.360386:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828416.360387:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787828416.360391:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c7014000 x1874670646727808/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828441 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787828416.360399:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828416.360401:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787828416.360421:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828416.360422:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828416.360424:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828416.360426:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787828416.360427:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787828416.360430:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828416.360433:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787828416.360435:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787828416.360438:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c7014000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646727808:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787828416.360442:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828416.360444:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828416.360445:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787828416.360447:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787828416.360449:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787828416.360450:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787828416.360452:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787828416.360454:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787828416.360456:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787828416.360457:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787828416.360459:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828416.360461:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828416.360462:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787828416.360463:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787828416.360465:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787828416.360466:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787828416.360468:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787828416.360469:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787828416.360470:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787828416.360472:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787828416.360473:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828416.360475:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828416.360478:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787828416.360480:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828416.360482:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787828416.360484:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787828416.360485:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1787828416.360487:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787828416.360488:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828416.360491:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787828416.360492:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787828416.360494:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828416.360498:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c7017840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646727936:192.168.203.143@tcp:8:kworker.0 00000100:00000001:0.0:1787828416.360501:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787828416.360504:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787828416.360505:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828416.360507:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1787828416.360510:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828416.360516:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787828416.360517:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369549888 : -126234340001728 : ffff8d30c7017840) 00000100:00000001:0.0:1787828416.360521:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1787828416.360532:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:0.0:1787828416.360536:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1787828416.360538:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828416.360539:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828416.360541:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787828416.360544:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828416.360547:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4297889316 00000100:00000001:0.0:1787828416.360550:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4297890560 00000100:00000001:0.0:1787828416.360552:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4297889316 00000100:00000001:0.0:1787828416.360555:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4297890560 00000100:00000001:0.0:1787828416.360557:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787828416.360559:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828416.360561:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787828416.360565:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828416/real 1787828416] req@ffff8d30c7017840 x1874670646727936/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828441 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787828416.360572:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787828416.360573:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787828416.360576:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787828416.360577:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787828416.360579:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828416.360581:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828416.360582:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787828416.360586:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c7017840 x1874670646727936/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828441 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787828416.360593:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828416.360594:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787828416.360597:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828416.360598:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828416.360599:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828416.360601:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787828416.360602:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787828416.360605:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828416.360607:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787828416.360609:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787828416.360612:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c7017840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646727936:192.168.203.143@tcp:8:kworker.0 00000100:00000001:0.0:1787828416.360616:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828416.360617:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828416.360619:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787828416.360621:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787828416.360622:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787828416.360624:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787828416.360625:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787828416.360627:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787828416.360628:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787828416.360630:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787828416.360657:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828416.360659:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828416.360662:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787828416.360664:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828416.360666:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787828416.360668:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828417.254728:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828417.254734:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787828417.254862:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787828417.254868:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787828417.254873:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828418.278775:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828418.278788:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787828418.278952:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787828418.278965:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787828418.278972:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828419.302747:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828419.302756:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787828419.302903:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1787828419.302909:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1787828419.302913:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828420.326768:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828420.326776:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787828420.326852:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787828420.326858:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787828420.326863:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828421.350741:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828421.350752:0:815:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1787828421.350755:0:815:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1787828421.350758:0:815:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828421.350765:0:815:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1787828421.350768:0:815:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828421.350770:0:815:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828421.350773:0:815:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787828421.350847:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787828421.350852:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787828421.350856:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787828421.478761:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828421.478780:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1787828421.478785:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787828421.478793:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787828421.478803:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787828421.478818:0:468:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 3221 00000100:00080000:3.0:1787828421.478839:0:468:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787828421.478845:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787828421.478847:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:3.0:1787828421.478854:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1787828421.478858:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1787828421.478864:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:3.0:1787828421.478868:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:3.0:1787828421.478871:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787828421.478873:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:3.0:1787828421.478880:0:468:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:3.0:1787828421.478883:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787828421.478887:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787828421.478892:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787828421.478897:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828421.478900:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:3.0:1787828421.478904:0:468:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787828421.478923:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787828421.478926:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828421.478931:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787828421.478935:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787828421.478942:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828421.478960:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787828421.478974:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c70170c0 x1874670646728064/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787828421.478999:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787828421.479005:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828421.479030:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787828421.479035:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828421.479040:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1787828421.479042:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787828421.479044:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787828421.479048:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787828421.479051:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3221 00000100:00080000:3.0:1787828421.479056:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787828421.479058:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787828421.479060:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:3.0:1787828421.479063:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1787828421.479067:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1787828421.479070:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000020:00000001:3.0:1787828421.479073:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:3.0:1787828421.479077:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787828421.479079:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:3.0:1787828421.479084:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:3.0:1787828421.479087:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787828421.479089:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787828421.479098:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787828421.479102:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828421.479104:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1787828421.479105:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:3.0:1787828421.479117:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00000001:0.0:1787828421.479142:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000008:00000001:3.0:1787828421.479145:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828421.479152:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828421.479154:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 02000000:00000001:3.0:1787828421.479159:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:0.0:1787828421.479164:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 02000000:00000001:3.0:1787828421.479164:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828421.479171:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:0.0:1787828421.479173:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787828421.479174:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787828421.479186:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828421.479199:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:3.0:1787828421.479200:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828421.479210:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00100000:3.0:1787828421.479217:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c7016d00 x1874670646728192/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:0.0:1787828421.479218:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828421.479237:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c70170c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646728064:192.168.203.143@tcp:250:kworker.0 00000100:00000001:3.0:1787828421.479242:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787828421.479249:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828421.479251:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:3.0:1787828421.479258:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828421.479261:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000100:00080000:3.0:1787828421.479265:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 02000000:00000001:0.0:1787828421.479267:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787828421.479275:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 02000000:00000001:0.0:1787828421.479284:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:3.0:1787828421.479285:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787828421.479292:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787828421.479296:0:468:0:(import.c:485:import_select_connection()) Process entered 02000000:00000001:0.0:1787828421.479298:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787828421.479302:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3221 00000100:00080000:3.0:1787828421.479311:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787828421.479316:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787828421.479318:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:3.0:1787828421.479321:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1787828421.479323:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1787828421.479326:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000020:00000001:3.0:1787828421.479329:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:0.0:1787828421.479330:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787828421.479333:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787828421.479337:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369547968 : -126234340003648 : ffff8d30c70170c0) 00000100:00000001:3.0:1787828421.479339:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:0.0:1787828421.479349:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00080000:3.0:1787828421.479350:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:3.0:1787828421.479359:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787828421.479361:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787828421.479366:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787828421.479369:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828421.479372:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:3.0:1787828421.479376:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1787828421.479380:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828421.479382:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787828421.479388:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787828421.479390:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828421.479392:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787828421.479395:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787828421.479398:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828421.479405:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787828421.479411:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c7014b40 x1874670646728320/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787828421.479422:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787828421.479425:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828421.479427:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:0.0:1787828421.479434:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00080000:3.0:1787828421.479435:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828421.479444:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1787828421.479447:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787828421.479449:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787828421.479454:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787828421.479460:0:468:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3221 00000100:00000001:0.0:1787828421.479461:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00080000:3.0:1787828421.479467:0:468:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787828421.479473:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787828421.479473:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787828421.479481:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:0.0:1787828421.479483:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828421.479486:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:0.0:1787828421.479490:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000020:00000001:3.0:1787828421.479492:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1787828421.479498:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000020:00000001:3.0:1787828421.479502:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:0.0:1787828421.479505:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828421.479506:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787828421.479510:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:0.0:1787828421.479517:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4297895680 00000100:00080000:3.0:1787828421.479518:0:468:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:3.0:1787828421.479529:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787828421.479531:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787828421.479535:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787828421.479537:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828421.479540:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1787828421.479541:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 00000008:00100000:3.0:1787828421.479543:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1787828421.479550:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828421.479553:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828421.479558:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4297895680 02000000:00000001:3.0:1787828421.479560:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787828421.479565:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828421.479569:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:0.0:1787828421.479573:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 02000000:00000001:3.0:1787828421.479574:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787828421.479584:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828421.479586:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787828421.479594:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828421.479598:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787828421.479646:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828421/real 1787828421] req@ffff8d30c70170c0 x1874670646728064/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828446 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787828421.479663:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000001:3.0:1787828421.479675:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:0.0:1787828421.479677:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787828421.479686:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787828421.479692:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00100000:3.0:1787828421.479696:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c70152c0 x1874670646728448/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787828421.479697:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828421.479706:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828421.479713:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787828421.479723:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787828421.479735:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828421.479743:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c70170c0 x1874670646728064/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828446 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787828421.479746:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828421.479764:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828421.479771:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787828421.479775:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828421.479778:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828421.479779:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828421.479782:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787828421.479785:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787828421.479790:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828421.479796:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787828421.479809:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787828421.479815:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c70170c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646728064:192.168.203.143@tcp:250:kworker.0 00000100:00000001:0.0:1787828421.479826:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828421.479830:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828421.479832:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787828421.479836:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787828421.479841:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787828421.479845:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787828421.479848:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787828421.479854:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787828421.479856:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787828421.479871:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787828421.479874:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828421.479877:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828421.479889:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787828421.479891:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828421.479894:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787828421.479897:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787828421.479899:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1787828421.479901:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787828421.479903:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828421.479905:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787828421.479908:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787828421.479911:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828421.479915:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c7016d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646728192:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787828421.479920:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787828421.479922:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787828421.479934:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828421.479937:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1787828421.479941:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828421.479949:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787828421.479951:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369547008 : -126234340004608 : ffff8d30c7016d00) 00000100:00000001:0.0:1787828421.479958:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1787828421.479979:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:0.0:1787828421.479988:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1787828421.479992:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828421.479995:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828421.479999:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787828421.480005:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828421.480026:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4297894423 00000100:00000001:0.0:1787828421.480033:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4297895680 00000100:00000001:0.0:1787828421.480039:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4297894423 00000100:00000001:0.0:1787828421.480043:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4297895680 00000100:00000001:0.0:1787828421.480048:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787828421.480052:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828421.480055:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787828421.480065:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828421/real 1787828421] req@ffff8d30c7016d00 x1874670646728192/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828446 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787828421.480083:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787828421.480086:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787828421.480091:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787828421.480095:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787828421.480098:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828421.480101:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828421.480104:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787828421.480113:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c7016d00 x1874670646728192/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828446 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787828421.480152:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828421.480156:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787828421.480159:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828421.480171:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828421.480173:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828421.480175:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787828421.480178:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787828421.480182:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828421.480186:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787828421.480190:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787828421.480194:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c7016d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646728192:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787828421.480203:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1787828421.480206:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787828421.480208:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828421.480211:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787828421.480213:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787828421.480216:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828421.480220:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c7014b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646728320:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787828421.480224:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787828421.480227:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787828421.480229:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828421.480232:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1787828421.480237:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828421.480244:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787828421.480246:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369538368 : -126234340013248 : ffff8d30c7014b40) 00000100:00000001:0.0:1787828421.480251:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1787828421.480280:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:0.0:1787828421.480286:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1787828421.480289:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828421.480291:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828421.480294:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787828421.480297:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828421.480302:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4297894423 00000100:00000001:0.0:1787828421.480305:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4297895680 00000100:00000001:0.0:1787828421.480308:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4297894423 00000100:00000001:0.0:1787828421.480312:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4297895680 00000100:00000001:0.0:1787828421.480314:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787828421.480317:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828421.480319:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787828421.480323:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828421/real 1787828421] req@ffff8d30c7014b40 x1874670646728320/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828446 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787828421.480333:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787828421.480336:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787828421.480339:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787828421.480341:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787828421.480342:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828421.480345:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828421.480347:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787828421.480352:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c7014b40 x1874670646728320/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828446 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787828421.480360:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828421.480362:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787828421.480365:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828421.480367:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828421.480368:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828421.480370:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787828421.480372:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787828421.480375:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828421.480379:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787828421.480381:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787828421.480386:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c7014b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646728320:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787828421.480390:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1787828421.480392:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787828421.480394:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828421.480396:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787828421.480398:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787828421.480401:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828421.480404:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c70152c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646728448:192.168.203.143@tcp:8:kworker.0 00000100:00000001:0.0:1787828421.480408:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787828421.480410:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787828421.480412:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828421.480414:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1787828421.480417:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828421.480422:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787828421.480424:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369540288 : -126234340011328 : ffff8d30c70152c0) 00000100:00000001:0.0:1787828421.480428:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1787828421.480439:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:0.0:1787828421.480444:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1787828421.480446:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828421.480448:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828421.480450:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787828421.480453:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828421.480457:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4297894436 00000100:00000001:0.0:1787828421.480461:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4297895680 00000100:00000001:0.0:1787828421.480463:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4297894436 00000100:00000001:0.0:1787828421.480466:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4297895680 00000100:00000001:0.0:1787828421.480469:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787828421.480471:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828421.480473:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787828421.480478:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828421/real 1787828421] req@ffff8d30c70152c0 x1874670646728448/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828446 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787828421.480487:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787828421.480488:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787828421.480490:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787828421.480492:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787828421.480494:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828421.480496:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828421.480497:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787828421.480502:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c70152c0 x1874670646728448/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828446 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787828421.480510:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828421.480512:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787828421.480516:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828421.480518:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828421.480520:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828421.480521:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787828421.480524:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787828421.480527:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828421.480531:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787828421.480534:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787828421.480537:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c70152c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646728448:192.168.203.143@tcp:8:kworker.0 00000100:00000001:0.0:1787828421.480541:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828421.480544:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828421.480545:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787828421.480547:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787828421.480550:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787828421.480552:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787828421.480553:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787828421.480556:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787828421.480557:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787828421.480560:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787828421.480562:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828421.480564:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828421.480565:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787828421.480566:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787828421.480569:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787828421.480570:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787828421.480572:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787828421.480574:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787828421.480576:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787828421.480578:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787828421.480580:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828421.480582:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828421.480583:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787828421.480585:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787828421.480586:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787828421.480588:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787828421.480589:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787828421.480591:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787828421.480592:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787828421.480594:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787828421.480596:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828421.480598:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828421.480632:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787828421.480634:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828421.480637:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787828421.480639:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828422.374728:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828422.374741:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787828422.374871:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1787828422.374878:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1787828422.374882:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828423.398725:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828423.398733:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787828423.398792:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787828423.398797:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787828423.398801:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828424.422717:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828424.422724:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787828424.422792:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787828424.422800:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787828424.422806:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828425.446757:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828425.446767:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787828425.446846:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1787828425.446853:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1787828425.446857:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828426.470735:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828426.470742:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787828426.470845:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787828426.470851:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787828426.470855:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787828426.598744:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828426.598760:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1787828426.598763:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787828426.598769:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787828426.598775:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787828426.598786:0:468:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 3227 00000100:00080000:3.0:1787828426.598810:0:468:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787828426.598815:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787828426.598818:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:3.0:1787828426.598826:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1787828426.598830:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1787828426.598835:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:3.0:1787828426.598851:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:3.0:1787828426.598854:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787828426.598857:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:3.0:1787828426.598863:0:468:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:3.0:1787828426.598866:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787828426.598869:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787828426.598872:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787828426.598876:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828426.598879:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:3.0:1787828426.598882:0:468:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787828426.598922:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787828426.598924:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828426.598928:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787828426.598931:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787828426.598937:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828426.598949:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787828426.598960:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c7014f00 x1874670646728576/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787828426.598973:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787828426.598977:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828426.598995:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787828426.599001:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828426.599007:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1787828426.599009:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787828426.599012:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787828426.599034:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787828426.599039:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3227 00000100:00080000:3.0:1787828426.599046:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787828426.599048:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1787828426.599049:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787828426.599052:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:0.0:1787828426.599055:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000020:00000001:3.0:1787828426.599055:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:0.0:1787828426.599058:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000020:00000001:3.0:1787828426.599058:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1787828426.599061:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:0.0:1787828426.599062:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000020:00000001:3.0:1787828426.599064:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:0.0:1787828426.599065:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828426.599067:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787828426.599069:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 02000000:00000001:0.0:1787828426.599071:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000100:00080000:3.0:1787828426.599074:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 02000000:00000001:0.0:1787828426.599075:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787828426.599079:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828426.599080:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787828426.599082:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00100000:0.0:1787828426.599117:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c7014f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646728576:192.168.203.143@tcp:250:kworker.0 02000000:00000001:3.0:1787828426.599119:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787828426.599123:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828426.599124:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:3.0:1787828426.599125:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 02000000:00000001:0.0:1787828426.599129:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000008:00100000:3.0:1787828426.599130:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 02000000:00000001:0.0:1787828426.599132:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:3.0:1787828426.599134:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828426.599136:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:3.0:1787828426.599137:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828426.599142:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787828426.599143:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787828426.599145:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828426.599147:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787828426.599149:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787828426.599152:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828426.599157:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828426.599159:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787828426.599162:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369539328 : -126234340012288 : ffff8d30c7014f00) 00000100:00100000:3.0:1787828426.599165:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c70143c0 x1874670646728704/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787828426.599167:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:3.0:1787828426.599183:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787828426.599185:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828426.599189:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787828426.599193:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828426.599215:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1787828426.599217:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787828426.599219:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787828426.599222:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787828426.599226:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3227 00000100:00080000:3.0:1787828426.599232:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787828426.599236:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787828426.599238:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:3.0:1787828426.599242:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1787828426.599244:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1787828426.599247:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:0.0:1787828426.599250:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828426.599251:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:3.0:1787828426.599254:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787828426.599256:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:0.0:1787828426.599257:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4297900800 00000100:00080000:3.0:1787828426.599261:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:3.0:1787828426.599265:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787828426.599267:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:0.0:1787828426.599269:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 02000000:00000001:3.0:1787828426.599271:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787828426.599274:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828426.599277:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4297900800 00000100:00000001:3.0:1787828426.599278:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1787828426.599282:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 00000008:00100000:3.0:1787828426.599282:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1787828426.599286:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828426.599288:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787828426.599289:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828426.599293:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787828426.599296:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787828426.599298:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828426.599299:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828426.599302:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787828426.599302:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787828426.599305:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787828426.599308:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828426.599313:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787828426.599314:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828426.599316:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:0.0:1787828426.599320:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787828426.599322:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00100000:3.0:1787828426.599323:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c7015680 x1874670646728832/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787828426.599324:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1787828426.599327:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787828426.599329:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828426.599333:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787828426.599336:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787828426.599337:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 02000000:00000001:0.0:1787828426.599341:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828426.599341:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828426.599346:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c70143c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646728704:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787828426.599346:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828426.599353:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787828426.599358:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787828426.599361:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787828426.599362:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 02000000:00000001:0.0:1787828426.599365:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00080000:3.0:1787828426.599367:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1787828426.599370:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 02000000:00000001:0.0:1787828426.599371:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787828426.599372:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787828426.599376:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:0.0:1787828426.599381:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00080000:3.0:1787828426.599382:0:468:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3227 00000100:00000001:0.0:1787828426.599384:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369536448 : -126234340015168 : ffff8d30c70143c0) 00000100:00000001:0.0:1787828426.599390:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00080000:3.0:1787828426.599390:0:468:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787828426.599393:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787828426.599395:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:3.0:1787828426.599399:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1787828426.599401:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1787828426.599404:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000020:00000001:3.0:1787828426.599408:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:0.0:1787828426.599410:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828426.599411:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787828426.599413:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:3.0:1787828426.599446:0:468:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:3.0:1787828426.599450:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828426.599452:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4297899542 02000000:00000001:3.0:1787828426.599453:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:0.0:1787828426.599456:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4297900800 02000000:00000001:3.0:1787828426.599457:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787828426.599459:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4297899542 02000000:00000001:3.0:1787828426.599460:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828426.599463:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4297900800 00000100:00000001:3.0:1787828426.599463:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1787828426.599467:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000008:00100000:3.0:1787828426.599467:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00000001:0.0:1787828426.599470:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:3.0:1787828426.599471:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828426.599473:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828426.599474:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828426.599476:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828426.599480:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 02000000:00000001:3.0:1787828426.599480:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:0.0:1787828426.599483:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 02000000:00000001:3.0:1787828426.599483:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828426.599486:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:0.0:1787828426.599489:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:3.0:1787828426.599489:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:0.0:1787828426.599492:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 02000000:00000001:3.0:1787828426.599492:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828426.599494:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1787828426.599496:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787828426.599498:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828426.599499:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828426.599501:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787828426.599504:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00100000:3.0:1787828426.599506:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c7015a40 x1874670646728960/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:0.0:1787828426.599509:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828426.599515:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c7015680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646728832:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787828426.599517:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787828426.599520:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:3.0:1787828426.599521:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828426.599533:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:3.0:1787828426.599534:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828426.599535:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828426.599538:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1787828426.599542:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828426.599550:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787828426.599552:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369541248 : -126234340010368 : ffff8d30c7015680) 00000100:00000001:0.0:1787828426.599557:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:0.0:1787828426.599574:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828426.599581:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4297899542 00000100:00000001:0.0:1787828426.599597:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4297900800 00000100:00000001:0.0:1787828426.599638:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4297899543 00000100:00000001:0.0:1787828426.599642:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4297900800 00000100:00000001:0.0:1787828426.599645:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787828426.599647:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828426.599649:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828426.599651:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828426.599664:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787828426.599665:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:0.0:1787828426.599667:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787828426.599668:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787828426.599670:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1787828426.599671:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787828426.599672:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828426.599674:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787828426.599676:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787828426.599678:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828426.599682:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c7015a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646728960:192.168.203.143@tcp:8:kworker.0 00000100:00000001:0.0:1787828426.599685:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787828426.599706:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787828426.599708:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828426.599710:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1787828426.599715:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828426.599725:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787828426.599727:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369542208 : -126234340009408 : ffff8d30c7015a40) 00000100:00000001:0.0:1787828426.599732:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:0.0:1787828426.599750:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828426.599756:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4297899556 00000100:00000001:0.0:1787828426.599760:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4297900800 00000100:00000001:0.0:1787828426.599763:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4297899556 00000100:00000001:0.0:1787828426.599766:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4297900800 00000100:00000001:0.0:1787828426.599769:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787828426.599773:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828426.599776:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828426.599780:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828426.599784:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787828426.599786:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:0.0:1787828426.599789:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787828426.599791:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787828426.599793:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828426.599795:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828427.494821:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828427.494834:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787828427.494963:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787828427.494969:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787828427.494972:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828428.518783:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828428.518794:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787828428.518943:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1787828428.518951:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1787828428.518957:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828429.542704:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828429.542714:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787828429.542865:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787828429.542875:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787828429.542883:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:2.0:1787828429.671047:0:2381:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.143:988 00000400:00000100:2.0:1787828429.671151:0:2381:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.143@tcp at host 192.168.203.143 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1787828429.671185:0:2381:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.143@tcp 00000100:00000001:2.0:1787828429.671223:0:2381:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1787828429.671284:0:2381:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828429.671290:0:2381:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828429.671295:0:2381:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:2.0:1787828429.671315:0:2381:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.143@tcp 00000100:00000001:2.0:1787828429.671328:0:2381:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1787828429.671335:0:2381:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828429.671338:0:2381:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828429.671342:0:2381:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:2.0:1787828429.671353:0:2381:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.143@tcp 00000100:00000001:0.0:1787828429.671361:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787828429.671364:0:2381:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1787828429.671374:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1787828429.671376:0:2381:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828429.671385:0:2381:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828429.671388:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00000001:2.0:1787828429.671390:0:2381:0:(events.c:68:request_out_callback()) Process leaving 00000100:00100000:0.0:1787828429.671417:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828426/real 1787828429] req@ffff8d30c7015680 x1874670646728832/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828451 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000100:2.0:1787828429.671418:0:2381:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.143@tcp 00000100:00000001:2.0:1787828429.671435:0:2381:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1787828429.671444:0:2381:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828429.671452:0:2381:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828429.671457:0:2381:0:(events.c:68:request_out_callback()) Process leaving 00000400:00000001:0.0:1787828429.671459:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787828429.671466:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787828429.671479:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787828429.671485:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787828429.671491:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828429.671497:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828429.671501:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787828429.671530:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c7015680 x1874670646728832/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828451 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787828429.671547:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828429.671553:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787828429.671561:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828429.671564:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828429.671567:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828429.671573:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787828429.671578:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787828429.671585:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828429.671596:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787828429.671657:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787828429.671667:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c7015680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646728832:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787828429.671681:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787828429.671693:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828426/real 1787828429] req@ffff8d30c70143c0 x1874670646728704/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828451 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787828429.671712:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787828429.671716:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787828429.671723:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787828429.671728:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787828429.671731:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828429.671735:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828429.671738:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787828429.671746:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c70143c0 x1874670646728704/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828451 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787828429.671759:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828429.671763:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787828429.671769:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828429.671772:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828429.671775:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828429.671807:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787828429.671811:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787828429.671818:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828429.671827:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787828429.671832:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787828429.671839:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c70143c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646728704:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787828429.671848:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787828429.671857:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828426/real 1787828429] req@ffff8d30c7014f00 x1874670646728576/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828451 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787828429.671875:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787828429.671877:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787828429.671884:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787828429.671888:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787828429.671890:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828429.671895:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828429.671897:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787828429.671907:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c7014f00 x1874670646728576/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828451 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787828429.671921:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828429.671925:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787828429.671930:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828429.671933:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828429.671936:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828429.671939:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787828429.671942:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787828429.671948:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828429.671954:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787828429.671959:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787828429.671965:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c7014f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646728576:192.168.203.143@tcp:250:kworker.0 00000100:00000001:0.0:1787828429.671973:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828429.671977:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828429.671981:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787828429.671986:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787828429.671996:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787828429.672001:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787828429.672006:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787828429.672014:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787828429.672018:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787828429.672024:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787828429.672026:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828429.672030:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828429.672032:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787828429.672034:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787828429.672038:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787828429.672040:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787828429.672042:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787828429.672046:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787828429.672048:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787828429.672051:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787828429.672054:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828429.672057:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828429.672060:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787828429.672062:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787828429.672093:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787828429.672096:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787828429.672098:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787828429.672101:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787828429.672103:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787828429.672107:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787828429.672109:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828429.672114:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828429.672118:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787828429.672121:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787828429.672124:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787828429.672134:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828426/real 1787828429] req@ffff8d30c7015a40 x1874670646728960/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828451 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787828429.672164:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787828429.672167:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787828429.672173:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787828429.672176:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787828429.672179:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828429.672183:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828429.672186:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787828429.672196:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c7015a40 x1874670646728960/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828451 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787828429.672210:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828429.672214:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787828429.672218:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828429.672221:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828429.672224:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828429.672227:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787828429.672230:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787828429.672235:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828429.672243:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787828429.672249:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787828429.672256:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c7015a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646728960:192.168.203.143@tcp:8:kworker.0 00000100:00000001:0.0:1787828429.672265:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828429.672270:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828429.672273:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787828429.672275:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787828429.672280:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787828429.672283:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787828429.672285:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787828429.672291:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787828429.672293:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787828429.672298:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787828429.672300:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828429.672305:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828429.672318:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787828429.672322:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828429.672327:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787828429.672330:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828430.566716:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828430.566727:0:815:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1787828430.566730:0:815:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1787828430.566734:0:815:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828430.566741:0:815:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1787828430.566744:0:815:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828430.566746:0:815:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828430.566748:0:815:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787828430.566838:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787828430.566846:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787828430.566852:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828431.590772:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828431.590782:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787828431.590920:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1787828431.590927:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1787828431.590933:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787828431.718736:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828431.718749:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1787828431.718752:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787828431.718757:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787828431.718763:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787828431.718772:0:468:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 3232 00000100:00080000:3.0:1787828431.718785:0:468:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787828431.718789:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787828431.718791:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:3.0:1787828431.718797:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1787828431.718801:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1787828431.718804:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:3.0:1787828431.718808:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:3.0:1787828431.718811:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787828431.718814:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:3.0:1787828431.718819:0:468:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:3.0:1787828431.718822:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787828431.718825:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787828431.718830:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787828431.718833:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828431.718836:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:3.0:1787828431.718839:0:468:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787828431.718863:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787828431.718865:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828431.718869:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787828431.718872:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787828431.718877:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828431.718907:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787828431.718920:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c7016580 x1874670646729088/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787828431.718933:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787828431.718939:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828431.718960:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787828431.718964:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828431.718969:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1787828431.718971:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787828431.718973:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787828431.718977:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787828431.718980:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3232 00000100:00080000:3.0:1787828431.718984:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787828431.718986:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787828431.718988:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:3.0:1787828431.718991:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1787828431.718993:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1787828431.718995:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000020:00000001:3.0:1787828431.718998:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:3.0:1787828431.719001:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787828431.719002:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:3.0:1787828431.719007:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:3.0:1787828431.719011:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787828431.719015:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787828431.719022:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787828431.719026:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828431.719047:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:3.0:1787828431.719079:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1787828431.719085:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828431.719087:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1787828431.719091:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828431.719104:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787828431.719114:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 02000000:00000001:3.0:1787828431.719116:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:0.0:1787828431.719120:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 02000000:00000001:3.0:1787828431.719123:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828431.719128:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828431.719131:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787828431.719141:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787828431.719147:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787828431.719149:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828431.719158:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787828431.719165:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828431.719169:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787828431.719184:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c70161c0 x1874670646729216/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787828431.719193:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c7016580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646729088:192.168.203.143@tcp:250:kworker.0 00000100:00000001:0.0:1787828431.719225:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:3.0:1787828431.719237:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 02000000:00000001:0.0:1787828431.719238:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787828431.719248:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828431.719250:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828431.719258:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:3.0:1787828431.719266:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828431.719274:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787828431.719276:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828431.719288:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1787828431.719291:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787828431.719294:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787828431.719300:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787828431.719308:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3232 00000100:00000001:0.0:1787828431.719313:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00080000:3.0:1787828431.719318:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787828431.719322:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369545088 : -126234340006528 : ffff8d30c7016580) 00000100:00000001:3.0:1787828431.719327:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787828431.719334:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:0.0:1787828431.719336:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000020:00000001:3.0:1787828431.719346:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1787828431.719350:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1787828431.719353:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000020:00000001:3.0:1787828431.719357:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:3.0:1787828431.719361:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787828431.719365:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:3.0:1787828431.719374:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:3.0:1787828431.719379:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787828431.719382:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787828431.719389:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787828431.719394:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828431.719398:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:3.0:1787828431.719401:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1787828431.719406:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828431.719409:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787828431.719418:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000800:00000100:0.0:1787828431.719426:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 02000000:00000001:3.0:1787828431.719427:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828431.719439:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787828431.719442:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787828431.719446:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828431.719456:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787828431.719459:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828431.719471:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828431.719484:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787828431.719486:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c7015a40 x1874670646729344/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787828431.719497:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787828431.719513:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828431.719514:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787828431.719525:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828431.719530:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4297905920 00000100:00000001:3.0:1787828431.719535:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787828431.719545:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828431.719551:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1787828431.719556:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:0.0:1787828431.719559:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 00000100:00080000:3.0:1787828431.719562:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787828431.719572:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:0.0:1787828431.719578:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4297905920 00000100:00080000:3.0:1787828431.719580:0:468:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3232 00000100:00000001:0.0:1787828431.719597:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 00000100:00080000:3.0:1787828431.719598:0:468:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787828431.719659:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787828431.719662:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:3.0:1787828431.719667:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1787828431.719671:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1787828431.719675:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000020:00000001:3.0:1787828431.719679:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:3.0:1787828431.719685:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787828431.719687:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:3.0:1787828431.719695:0:468:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:3.0:1787828431.719708:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828431.719710:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787828431.719715:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:0.0:1787828431.719723:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787828431.719726:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787828431.719734:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 02000000:00000001:3.0:1787828431.719736:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828431.719745:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:3.0:1787828431.719750:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1787828431.719760:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828431/real 1787828431] req@ffff8d30c7016580 x1874670646729088/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828456 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00000001:3.0:1787828431.719761:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828431.719772:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787828431.719789:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787828431.719793:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828431.719805:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787828431.719810:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000400:00000001:0.0:1787828431.719814:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 02000000:00000001:3.0:1787828431.719817:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:0.0:1787828431.719822:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787828431.719833:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828431.719837:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787828431.719847:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787828431.719856:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787828431.719858:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c7014f00 x1874670646729472/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787828431.719866:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828431.719872:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787828431.719886:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c7016580 x1874670646729088/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828456 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787828431.719905:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828431.719911:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787828431.719918:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828431.719921:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828431.719924:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828431.719929:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787828431.719933:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787828431.719940:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828431.719950:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787828431.719956:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787828431.719965:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c7016580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646729088:192.168.203.143@tcp:250:kworker.0 00000100:00000001:0.0:1787828431.719976:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828431.719981:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828431.719984:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787828431.719989:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787828431.719997:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787828431.720003:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787828431.720007:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787828431.720015:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787828431.720020:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787828431.720025:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787828431.720028:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828431.720033:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828431.720070:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787828431.720077:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828431.720082:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787828431.720086:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787828431.720089:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1787828431.720093:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787828431.720096:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828431.720101:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787828431.720105:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787828431.720110:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828431.720118:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c70161c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646729216:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787828431.720125:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787828431.720130:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787828431.720133:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828431.720137:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1787828431.720144:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828431.720158:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787828431.720162:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369544128 : -126234340007488 : ffff8d30c70161c0) 00000100:00000001:0.0:1787828431.720171:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1787828431.720218:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:0.0:1787828431.720231:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1787828431.720240:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828431.720244:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828431.720252:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787828431.720263:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828431.720285:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4297904663 00000100:00000001:3.0:1787828431.720287:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787828431.720297:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828431.720300:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4297905920 00000100:00000001:3.0:1787828431.720310:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828431.720311:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4297904663 00000100:00000001:0.0:1787828431.720330:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4297905920 00000100:00000001:0.0:1787828431.720337:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787828431.720341:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828431.720347:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787828431.720360:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828431/real 1787828431] req@ffff8d30c70161c0 x1874670646729216/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828456 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787828431.720380:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787828431.720385:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787828431.720393:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787828431.720399:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787828431.720403:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828431.720408:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828431.720411:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787828431.720425:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c70161c0 x1874670646729216/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828456 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787828431.720442:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828431.720447:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787828431.720454:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828431.720457:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828431.720459:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828431.720463:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787828431.720468:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787828431.720473:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828431.720482:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787828431.720489:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787828431.720498:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c70161c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646729216:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787828431.720508:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1787828431.720512:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787828431.720514:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828431.720519:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787828431.720523:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787828431.720529:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828431.720536:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c7015a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646729344:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787828431.720549:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787828431.720555:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787828431.720558:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828431.720563:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1787828431.720571:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828431.720584:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787828431.720589:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369542208 : -126234340009408 : ffff8d30c7015a40) 00000100:00000001:0.0:1787828431.720598:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1787828431.720693:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:0.0:1787828431.720706:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1787828431.720715:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828431.720720:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828431.720724:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787828431.720731:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828431.720740:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4297904664 00000100:00000001:0.0:1787828431.720745:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4297905920 00000100:00000001:0.0:1787828431.720750:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4297904664 00000100:00000001:0.0:1787828431.720754:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4297905920 00000100:00000001:0.0:1787828431.720759:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787828431.720763:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828431.720767:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787828431.720776:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828431/real 1787828431] req@ffff8d30c7015a40 x1874670646729344/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828456 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787828431.720794:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787828431.720797:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787828431.720803:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787828431.720806:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787828431.720809:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828431.720823:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828431.720826:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787828431.720834:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c7015a40 x1874670646729344/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828456 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787828431.720862:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828431.720865:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787828431.720868:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828431.720871:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828431.720873:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828431.720877:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787828431.720880:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787828431.720886:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828431.720892:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787828431.720897:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787828431.720903:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c7015a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646729344:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787828431.720912:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828431.720917:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828431.720922:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787828431.720926:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787828431.720932:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787828431.720937:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787828431.720940:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787828431.720947:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787828431.720953:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787828431.720958:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787828431.720960:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828431.720964:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828431.720966:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787828431.720968:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787828431.720972:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787828431.720974:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787828431.720976:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787828431.720981:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787828431.720983:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787828431.720987:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787828431.720990:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828431.721011:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828431.721019:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787828431.721022:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828431.721026:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787828431.721029:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787828431.721031:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1787828431.721034:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787828431.721036:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828431.721041:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787828431.721044:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787828431.721048:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828431.721090:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c7014f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646729472:192.168.203.143@tcp:8:kworker.0 00000100:00000001:0.0:1787828431.721096:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787828431.721102:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787828431.721105:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828431.721108:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1787828431.721113:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828431.721134:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787828431.721136:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369539328 : -126234340012288 : ffff8d30c7014f00) 00000100:00000001:0.0:1787828431.721141:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1787828431.721164:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:0.0:1787828431.721172:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1787828431.721175:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828431.721177:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828431.721180:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787828431.721183:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828431.721187:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4297904677 00000100:00000001:0.0:1787828431.721190:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4297905920 00000100:00000001:0.0:1787828431.721193:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4297904677 00000100:00000001:0.0:1787828431.721197:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4297905920 00000100:00000001:0.0:1787828431.721200:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787828431.721203:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828431.721205:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787828431.721211:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828431/real 1787828431] req@ffff8d30c7014f00 x1874670646729472/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828456 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787828431.721227:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787828431.721229:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787828431.721232:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787828431.721234:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787828431.721235:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828431.721237:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828431.721239:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787828431.721243:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c7014f00 x1874670646729472/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828456 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787828431.721250:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828431.721252:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787828431.721255:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828431.721256:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828431.721257:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828431.721259:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787828431.721261:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787828431.721263:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828431.721266:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787828431.721268:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787828431.721272:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c7014f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646729472:192.168.203.143@tcp:8:kworker.0 00000100:00000001:0.0:1787828431.721276:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828431.721278:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828431.721279:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787828431.721281:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787828431.721283:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787828431.721285:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787828431.721286:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787828431.721289:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787828431.721290:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787828431.721292:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787828431.721294:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828431.721296:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828431.721300:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787828431.721301:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828431.721304:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787828431.721305:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828432.614739:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828432.614748:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787828432.614929:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787828432.614935:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787828432.614939:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828433.638779:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828433.638789:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787828433.638889:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787828433.638895:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787828433.638899:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828434.662702:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828434.662709:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787828434.662801:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1787828434.662806:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1787828434.662822:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828435.686821:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828435.686831:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787828435.686981:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787828435.686986:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787828435.686990:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828436.710697:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828436.710709:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787828436.710791:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787828436.710797:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787828436.710801:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787828436.838889:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828436.838901:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1787828436.838905:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787828436.838909:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787828436.838938:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787828436.838949:0:468:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 3237 00000100:00080000:0.0:1787828436.838976:0:468:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787828436.838981:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787828436.839000:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787828436.839008:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787828436.839032:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787828436.839039:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:0.0:1787828436.839044:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:0.0:1787828436.839048:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787828436.839061:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:0.0:1787828436.839083:0:468:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787828436.839089:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828436.839094:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787828436.839101:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787828436.839107:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828436.839111:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1787828436.839127:0:468:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828436.839145:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787828436.839149:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828436.839155:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787828436.839161:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787828436.839168:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828436.839185:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828436.839199:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d270c0 x1874670646729600/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787828436.839213:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787828436.839219:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828436.839245:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787828436.839250:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828436.839254:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1787828436.839256:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787828436.839259:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787828436.839263:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787828436.839266:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3237 00000100:00080000:0.0:1787828436.839272:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787828436.839275:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787828436.839277:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787828436.839280:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787828436.839282:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787828436.839284:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000020:00000001:0.0:1787828436.839287:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:0.0:1787828436.839290:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787828436.839293:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:0.0:1787828436.839298:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787828436.839303:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828436.839307:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787828436.839316:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787828436.839321:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828436.839326:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:1.0:1787828436.839326:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787828436.839336:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000008:00100000:0.0:1787828436.839340:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00000001:1.0:1787828436.839341:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787828436.839348:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000008:00000001:0.0:1787828436.839354:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828436.839354:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828436.839367:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828436.839368:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787828436.839375:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787828436.839381:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828436.839383:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00100000:1.0:1787828436.839391:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d270c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646729600:192.168.203.143@tcp:250:kworker.0 02000000:00000001:0.0:1787828436.839392:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828436.839406:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:1.0:1787828436.839407:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787828436.839415:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1787828436.839417:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787828436.839422:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828436.839423:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828436.839434:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:0.0:1787828436.839439:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828436.839442:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828436.839455:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d24000 x1874670646729728/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787828436.839475:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787828436.839478:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787828436.839480:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828436.839485:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601348800 : -126234108202816 : ffff8d30d4d270c0) 00000100:00000001:0.0:1787828436.839491:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828436.839493:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00080000:0.0:1787828436.839501:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828436.839508:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1787828436.839510:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787828436.839513:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787828436.839517:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787828436.839531:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3237 00000100:00080000:0.0:1787828436.839552:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787828436.839559:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000800:00000100:1.0:1787828436.839560:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:0.0:1787828436.839563:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787828436.839567:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787828436.839568:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787828436.839571:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000020:00000001:0.0:1787828436.839574:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:0.0:1787828436.839576:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787828436.839582:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:0.0:1787828436.839597:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:1.0:1787828436.839670:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787828436.839677:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787828436.839683:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828436.839684:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828436.839688:0:2390:0:(events.c:68:request_out_callback()) Process leaving 02000000:00000001:0.0:1787828436.839692:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1787828436.839699:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828436.839700:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787828436.839710:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828436.839711:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4297911041 00000100:00000001:0.0:1787828436.839715:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1787828436.839738:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1787828436.839741:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828436.839744:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828436.839746:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 02000000:00000001:0.0:1787828436.839752:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:1.0:1787828436.839756:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4297911041 02000000:00000001:0.0:1787828436.839758:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828436.839766:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1787828436.839767:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787828436.839775:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:1.0:1787828436.839788:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787828436.839789:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828436.839794:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828436.839817:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00000001:0.0:1787828436.839824:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787828436.839830:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828436/real 1787828436] req@ffff8d30d4d270c0 x1874670646729600/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828461 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828436.839847:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d24f00 x1874670646729856/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000001:1.0:1787828436.839858:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787828436.839863:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787828436.839872:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787828436.839896:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787828436.839897:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787828436.839903:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828436.839904:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828436.839910:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828436.839914:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828436.839915:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:0.0:1787828436.839920:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1787828436.839928:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d270c0 x1874670646729600/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828461 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828436.839929:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1787828436.839935:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787828436.839939:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787828436.839948:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787828436.839972:0:468:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3237 00000100:00080000:0.0:1787828436.839979:0:468:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1787828436.839985:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828436.839986:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787828436.839994:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:1.0:1787828436.839997:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000020:00000001:0.0:1787828436.840000:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:1.0:1787828436.840006:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1787828436.840007:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:1.0:1787828436.840039:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000020:00000001:0.0:1787828436.840041:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:1.0:1787828436.840043:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1787828436.840052:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:1.0:1787828436.840053:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787828436.840059:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787828436.840062:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787828436.840068:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:1.0:1787828436.840069:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828436.840083:0:468:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00080000:1.0:1787828436.840085:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787828436.840094:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000001:0.0:1787828436.840096:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828436.840104:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00100000:1.0:1787828436.840105:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d270c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646729600:192.168.203.143@tcp:250:kworker.0 02000000:00000001:0.0:1787828436.840111:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787828436.840115:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:0.0:1787828436.840118:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828436.840122:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828436.840126:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:1.0:1787828436.840128:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 00000008:00100000:0.0:1787828436.840133:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 02000000:00000001:1.0:1787828436.840135:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 00000008:00000001:0.0:1787828436.840143:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828436.840144:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00000001:0.0:1787828436.840147:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787828436.840151:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787828436.840156:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787828436.840157:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787828436.840162:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828436.840163:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 00000100:00000001:0.0:1787828436.840167:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1787828436.840168:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 02000000:00000001:0.0:1787828436.840172:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787828436.840178:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828436.840178:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787828436.840183:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828436.840189:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828436.840191:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828436.840199:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787828436.840206:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828436.840207:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d25a40 x1874670646729984/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787828436.840212:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787828436.840217:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1787828436.840223:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787828436.840227:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787828436.840233:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787828436.840234:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828436.840242:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828436.840243:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787828436.840251:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787828436.840256:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828436.840257:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787828436.840271:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d24000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646729728:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787828436.840275:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787828436.840278:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787828436.840281:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828436.840283:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787828436.840289:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828436.840298:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787828436.840300:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601336320 : -126234108215296 : ffff8d30d4d24000) 00000100:00000001:1.0:1787828436.840306:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1787828436.840361:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787828436.840373:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787828436.840378:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787828436.840380:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828436.840383:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787828436.840388:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828436.840394:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4297909783 00000100:00000001:1.0:1787828436.840399:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4297911041 00000100:00000001:1.0:1787828436.840403:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4297909783 00000100:00000001:1.0:1787828436.840407:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4297911041 00000100:00000001:1.0:1787828436.840410:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787828436.840413:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828436.840416:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787828436.840424:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828436/real 1787828436] req@ffff8d30d4d24000 x1874670646729728/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828461 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787828436.840436:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787828436.840438:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787828436.840441:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787828436.840443:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787828436.840445:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828436.840447:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787828436.840449:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787828436.840455:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d24000 x1874670646729728/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828461 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787828436.840475:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828436.840478:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787828436.840481:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828436.840483:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787828436.840484:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828436.840486:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787828436.840489:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787828436.840492:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828436.840495:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787828436.840498:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787828436.840502:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d24000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646729728:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787828436.840506:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787828436.840507:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787828436.840509:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828436.840512:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787828436.840514:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787828436.840517:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787828436.840520:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d24f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646729856:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787828436.840524:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787828436.840526:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787828436.840528:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828436.840530:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787828436.840540:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828436.840546:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787828436.840547:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601340160 : -126234108211456 : ffff8d30d4d24f00) 00000100:00000001:1.0:1787828436.840551:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1787828436.840564:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787828436.840568:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787828436.840571:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787828436.840573:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828436.840576:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787828436.840579:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828436.840583:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4297909783 00000100:00000001:1.0:1787828436.840586:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4297911041 00000100:00000001:1.0:1787828436.840588:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4297909783 00000100:00000001:1.0:1787828436.840591:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4297911041 00000100:00000001:1.0:1787828436.840594:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787828436.840596:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828436.840636:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787828436.840641:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828436/real 1787828436] req@ffff8d30d4d24f00 x1874670646729856/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828461 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787828436.840651:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787828436.840652:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787828436.840655:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787828436.840657:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787828436.840659:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828436.840673:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787828436.840675:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787828436.840679:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d24f00 x1874670646729856/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828461 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787828436.840688:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828436.840690:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787828436.840693:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828436.840695:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787828436.840696:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828436.840698:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787828436.840700:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787828436.840703:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828436.840706:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787828436.840709:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787828436.840714:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d24f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646729856:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787828436.840719:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828436.840721:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787828436.840723:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787828436.840725:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787828436.840727:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787828436.840747:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787828436.840748:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787828436.840751:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787828436.840752:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787828436.840755:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787828436.840757:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828436.840758:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787828436.840760:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787828436.840761:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787828436.840763:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787828436.840764:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787828436.840766:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787828436.840767:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787828436.840768:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787828436.840771:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787828436.840772:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828436.840774:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828436.840777:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787828436.840779:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828436.840784:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787828436.840786:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1787828436.840787:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787828436.840789:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787828436.840791:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828436.840793:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787828436.840816:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787828436.840818:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787828436.840822:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d25a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646729984:192.168.203.143@tcp:8:kworker.0 00000100:00000001:1.0:1787828436.840826:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787828436.840829:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787828436.840831:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828436.840833:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787828436.840837:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828436.840855:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787828436.840857:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601343040 : -126234108208576 : ffff8d30d4d25a40) 00000100:00000001:1.0:1787828436.840861:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1787828436.840894:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787828436.840898:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787828436.840900:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787828436.840902:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828436.840904:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787828436.840907:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828436.840911:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4297909797 00000100:00000001:1.0:1787828436.840913:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4297911041 00000100:00000001:1.0:1787828436.840916:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4297909797 00000100:00000001:1.0:1787828436.840919:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4297911041 00000100:00000001:1.0:1787828436.840921:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787828436.840923:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828436.840925:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787828436.840929:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828436/real 1787828436] req@ffff8d30d4d25a40 x1874670646729984/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828461 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787828436.840937:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787828436.840938:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787828436.840941:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787828436.840959:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787828436.840961:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828436.840964:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787828436.840966:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787828436.840970:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d25a40 x1874670646729984/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828461 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787828436.840984:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828436.840987:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787828436.840990:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828436.840991:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787828436.840994:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828436.840996:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787828436.840998:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787828436.841002:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828436.841005:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787828436.841008:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787828436.841011:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d25a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646729984:192.168.203.143@tcp:8:kworker.0 00000100:00000001:1.0:1787828436.841064:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828436.841066:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787828436.841067:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787828436.841069:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787828436.841071:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787828436.841072:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787828436.841074:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787828436.841076:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787828436.841078:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787828436.841080:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787828436.841082:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828436.841084:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828436.841087:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787828436.841089:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828436.841091:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787828436.841092:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828437.734739:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828437.734747:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787828437.734841:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1787828437.734847:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1787828437.734852:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828438.758774:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828438.758784:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787828438.758872:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787828438.758878:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787828438.758882:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828439.782707:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828439.782715:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787828439.782860:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787828439.782869:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787828439.782875:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828440.806758:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828440.806771:0:815:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1787828440.806775:0:815:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1787828440.806780:0:815:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828440.806788:0:815:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1787828440.806791:0:815:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828440.806794:0:815:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828440.806798:0:815:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787828440.806958:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1787828440.806965:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1787828440.806970:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828441.830767:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828441.830778:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787828441.830910:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787828441.830918:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787828441.830924:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787828441.958770:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828441.958786:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1787828441.958790:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787828441.958797:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787828441.958804:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787828441.958814:0:40701:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 3242 00000100:00080000:2.0:1787828441.958832:0:40701:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787828441.958837:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828441.958841:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787828441.958847:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787828441.958851:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787828441.958856:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:2.0:1787828441.958859:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:2.0:1787828441.958862:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828441.958864:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:2.0:1787828441.958870:0:40701:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787828441.958874:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828441.958878:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787828441.958883:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828441.958888:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828441.958891:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:2.0:1787828441.958894:0:40701:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828441.958915:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787828441.958917:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828441.958922:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787828441.958926:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787828441.958933:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828441.958952:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787828441.958966:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2f0c0 x1874670646730112/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787828441.959007:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787828441.959014:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828441.959039:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787828441.959044:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828441.959048:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1787828441.959050:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787828441.959053:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787828441.959056:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787828441.959059:0:40701:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3242 00000100:00080000:2.0:1787828441.959064:0:40701:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787828441.959066:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828441.959068:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787828441.959071:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787828441.959073:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787828441.959076:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000020:00000001:2.0:1787828441.959079:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:2.0:1787828441.959082:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828441.959086:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:2.0:1787828441.959090:0:40701:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787828441.959093:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828441.959095:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787828441.959103:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828441.959107:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828441.959111:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1787828441.959119:0:40701:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1787828441.959124:0:40701:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828441.959126:0:40701:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828441.959133:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:2.0:1787828441.959134:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787828441.959143:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828441.959148:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787828441.959150:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:1.0:1787828441.959152:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 02000000:00000001:2.0:1787828441.959154:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828441.959162:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1787828441.959164:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828441.959172:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00100000:2.0:1787828441.959177:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2c3c0 x1874670646730240/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787828441.959182:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828441.959207:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 02000000:00000001:1.0:1787828441.959208:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:2.0:1787828441.959213:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828441.959220:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787828441.959222:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787828441.959235:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828441.959239:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1787828441.959241:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787828441.959243:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787828441.959246:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787828441.959249:0:40701:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3242 02000000:00000001:1.0:1787828441.959258:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787828441.959259:0:40701:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787828441.959276:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828441.959279:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787828441.959288:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000100:00100000:1.0:1787828441.959290:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646730112:192.168.203.143@tcp:250:kworker.0 00000020:00000001:2.0:1787828441.959291:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787828441.959294:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000020:00000001:2.0:1787828441.959308:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:1.0:1787828441.959311:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:2.0:1787828441.959315:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828441.959318:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 02000000:00000001:1.0:1787828441.959320:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000100:00080000:2.0:1787828441.959327:0:40701:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 02000000:00000001:1.0:1787828441.959328:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828441.959347:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828441.959348:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1787828441.959359:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787828441.959365:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787828441.959369:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828441.959370:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828441.959384:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1787828441.959387:0:40701:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1787828441.959392:0:40701:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828441.959394:0:40701:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828441.959403:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:1.0:1787828441.959405:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787828441.959413:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417880768 : -126234291670848 : ffff8d30c9e2f0c0) 02000000:00000001:2.0:1787828441.959413:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828441.959428:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787828441.959430:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787828441.959432:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828441.959444:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:2.0:1787828441.959445:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787828441.959461:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2e580 x1874670646730368/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787828441.959471:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787828441.959474:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828441.959477:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787828441.959482:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828441.959486:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1787828441.959488:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787828441.959490:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787828441.959492:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787828441.959496:0:40701:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3242 00000100:00080000:2.0:1787828441.959501:0:40701:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787828441.959504:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828441.959506:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787828441.959508:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787828441.959511:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787828441.959515:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000020:00000001:2.0:1787828441.959518:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:2.0:1787828441.959527:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828441.959534:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000800:00000100:1.0:1787828441.959535:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00080000:2.0:1787828441.959550:0:40701:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787828441.959554:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828441.959556:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787828441.959560:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828441.959562:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828441.959566:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1787828441.959569:0:40701:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000008:00000001:2.0:1787828441.959573:0:40701:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828441.959580:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787828441.959588:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828441.959591:0:40701:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828441.959592:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828441.959658:0:2390:0:(events.c:68:request_out_callback()) Process leaving 02000000:00000001:2.0:1787828441.959659:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787828441.959663:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828441.959668:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787828441.959671:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:1.0:1787828441.959672:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828441.959680:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828441.959686:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4297916161 00000100:00000001:2.0:1787828441.959693:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828441.959704:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 00000100:00100000:2.0:1787828441.959706:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2f480 x1874670646730496/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787828441.959721:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787828441.959726:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828441.959727:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4297916161 00000100:00000001:2.0:1787828441.959736:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828441.959740:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 00000100:00000001:1.0:1787828441.959755:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787828441.959762:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828441.959766:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787828441.959781:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828441/real 1787828441] req@ffff8d30c9e2f0c0 x1874670646730112/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828466 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787828441.959798:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787828441.959800:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787828441.959805:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787828441.959809:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787828441.959812:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828441.959815:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787828441.959819:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787828441.959825:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2f0c0 x1874670646730112/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828466 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787828441.959835:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828441.959840:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787828441.959843:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828441.959845:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787828441.959847:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828441.959850:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787828441.959852:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787828441.959857:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828441.959862:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787828441.959865:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787828441.959870:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646730112:192.168.203.143@tcp:250:kworker.0 00000100:00000001:1.0:1787828441.959877:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828441.959880:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787828441.959882:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787828441.959885:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787828441.959890:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787828441.959893:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787828441.959896:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787828441.959901:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787828441.959903:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787828441.959908:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787828441.959910:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828441.959913:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828441.959923:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787828441.959925:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828441.959929:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787828441.959931:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1787828441.959933:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787828441.959935:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787828441.959938:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828441.959941:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787828441.959944:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787828441.959948:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787828441.959952:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646730240:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787828441.959956:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787828441.959959:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787828441.959962:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828441.959964:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787828441.959968:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828441.959975:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787828441.960002:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417869248 : -126234291682368 : ffff8d30c9e2c3c0) 00000100:00000001:1.0:1787828441.960010:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1787828441.960034:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787828441.960040:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787828441.960043:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787828441.960045:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828441.960047:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787828441.960051:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828441.960056:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4297914903 00000100:00000001:1.0:1787828441.960060:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4297916161 00000100:00000001:1.0:1787828441.960063:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4297914903 00000100:00000001:1.0:1787828441.960066:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4297916161 00000100:00000001:1.0:1787828441.960069:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787828441.960071:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828441.960074:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787828441.960079:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828441/real 1787828441] req@ffff8d30c9e2c3c0 x1874670646730240/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828466 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787828441.960090:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787828441.960091:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787828441.960094:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787828441.960096:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787828441.960098:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828441.960100:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787828441.960102:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787828441.960107:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2c3c0 x1874670646730240/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828466 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787828441.960117:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828441.960121:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787828441.960125:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828441.960128:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787828441.960131:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828441.960135:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787828441.960139:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787828441.960144:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828441.960150:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787828441.960155:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787828441.960161:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646730240:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787828441.960169:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787828441.960173:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787828441.960176:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828441.960180:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787828441.960184:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787828441.960188:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787828441.960195:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646730368:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787828441.960202:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787828441.960207:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787828441.960211:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828441.960215:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787828441.960221:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828441.960231:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787828441.960234:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417877888 : -126234291673728 : ffff8d30c9e2e580) 00000100:00000001:1.0:1787828441.960239:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1787828441.960258:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787828441.960264:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787828441.960267:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787828441.960269:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828441.960272:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787828441.960276:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828441.960281:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4297914903 00000100:00000001:1.0:1787828441.960284:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4297916161 00000100:00000001:1.0:1787828441.960287:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4297914903 00000100:00000001:1.0:1787828441.960290:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4297916161 00000100:00000001:1.0:1787828441.960292:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787828441.960295:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828441.960297:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787828441.960302:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828441/real 1787828441] req@ffff8d30c9e2e580 x1874670646730368/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828466 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787828441.960312:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787828441.960313:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787828441.960316:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787828441.960318:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787828441.960320:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828441.960322:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787828441.960324:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787828441.960329:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2e580 x1874670646730368/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828466 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787828441.960338:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828441.960341:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787828441.960344:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828441.960346:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787828441.960347:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828441.960349:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787828441.960351:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787828441.960354:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828441.960358:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787828441.960360:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787828441.960365:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2e580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646730368:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787828441.960370:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787828441.960372:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787828441.960374:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828441.960377:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787828441.960379:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787828441.960381:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787828441.960385:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646730496:192.168.203.143@tcp:8:kworker.0 00000100:00000001:1.0:1787828441.960388:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787828441.960391:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787828441.960394:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828441.960396:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787828441.960400:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828441.960405:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787828441.960407:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417881728 : -126234291669888 : ffff8d30c9e2f480) 00000100:00000001:1.0:1787828441.960411:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1787828441.960422:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787828441.960427:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787828441.960430:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787828441.960433:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828441.960436:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787828441.960442:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828441.960448:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4297914916 00000100:00000001:1.0:1787828441.960453:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4297916161 00000100:00000001:1.0:1787828441.960457:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4297914916 00000100:00000001:1.0:1787828441.960461:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4297916161 00000100:00000001:1.0:1787828441.960465:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787828441.960469:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828441.960472:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787828441.960481:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828441/real 1787828441] req@ffff8d30c9e2f480 x1874670646730496/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828466 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787828441.960513:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787828441.960516:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787828441.960521:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787828441.960524:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787828441.960527:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828441.960530:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787828441.960533:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787828441.960541:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2f480 x1874670646730496/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828466 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787828441.960550:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828441.960553:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787828441.960556:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828441.960558:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787828441.960559:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828441.960561:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787828441.960563:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787828441.960566:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828441.960570:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787828441.960572:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787828441.960576:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646730496:192.168.203.143@tcp:8:kworker.0 00000100:00000001:1.0:1787828441.960581:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828441.960584:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787828441.960586:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787828441.960588:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787828441.960591:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787828441.960593:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787828441.960595:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787828441.960630:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787828441.960633:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787828441.960637:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787828441.960638:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828441.960642:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787828441.960643:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787828441.960645:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787828441.960648:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787828441.960650:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787828441.960651:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787828441.960653:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787828441.960654:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787828441.960658:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787828441.960659:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828441.960661:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787828441.960663:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787828441.960664:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787828441.960666:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787828441.960667:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787828441.960668:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787828441.960670:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787828441.960671:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787828441.960673:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787828441.960675:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828441.960677:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828441.960682:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787828441.960684:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828441.960687:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787828441.960689:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828442.854766:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828442.854775:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787828442.854886:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787828442.854894:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787828442.854900:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828443.878741:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828443.878750:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787828443.878816:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1787828443.878821:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1787828443.878825:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828444.902806:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828444.902834:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787828444.902911:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787828444.902919:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787828444.902924:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828445.926700:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828445.926707:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787828445.926773:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787828445.926779:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787828445.926803:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828446.950769:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828446.950776:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787828446.950899:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1787828446.950904:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1787828446.950907:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787828447.078684:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828447.078706:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1787828447.078709:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787828447.078713:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787828447.078719:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787828447.078726:0:40701:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 3247 00000100:00080000:2.0:1787828447.078738:0:40701:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787828447.078741:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828447.078743:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787828447.078746:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787828447.078749:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787828447.078752:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:2.0:1787828447.078754:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:2.0:1787828447.078756:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828447.078757:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:2.0:1787828447.078761:0:40701:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787828447.078763:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828447.078765:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787828447.078769:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828447.078771:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828447.078773:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:2.0:1787828447.078775:0:40701:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828447.078800:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787828447.078802:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828447.078805:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787828447.078807:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787828447.078812:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828447.078827:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787828447.078836:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2ed00 x1874670646730624/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787828447.078845:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787828447.078849:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828447.078876:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787828447.078879:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828447.078884:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1787828447.078885:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787828447.078886:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787828447.078888:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787828447.078890:0:40701:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3247 00000100:00080000:2.0:1787828447.078892:0:40701:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787828447.078893:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828447.078894:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787828447.078896:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787828447.078896:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787828447.078898:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000020:00000001:2.0:1787828447.078900:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:2.0:1787828447.078901:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828447.078903:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:2.0:1787828447.078905:0:40701:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787828447.078906:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828447.078908:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787828447.078913:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828447.078915:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828447.078916:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1787828447.078920:0:40701:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1787828447.078923:0:40701:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828447.078924:0:40701:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828447.078928:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787828447.078929:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828447.078930:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787828447.078931:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787828447.078933:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828447.078938:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787828447.078956:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2da40 x1874670646730752/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787828447.078978:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787828447.078980:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828447.078982:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787828447.078984:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828447.078987:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1787828447.078989:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:1.0:1787828447.078990:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00080000:2.0:1787828447.078993:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787828447.078997:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:1.0:1787828447.078999:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00080000:2.0:1787828447.079005:0:40701:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3247 00000100:00000001:1.0:1787828447.079007:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787828447.079013:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00080000:2.0:1787828447.079013:0:40701:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1787828447.079020:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828447.079021:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828447.079027:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787828447.079041:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787828447.079045:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 02000000:00000001:1.0:1787828447.079050:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000020:00000001:2.0:1787828447.079050:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000020:00000001:2.0:1787828447.079059:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 02000000:00000001:1.0:1787828447.079060:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787828447.079072:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 02000000:00000001:1.0:1787828447.079073:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828447.079080:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:2.0:1787828447.079089:0:40701:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00100000:1.0:1787828447.079090:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646730624:192.168.203.143@tcp:250:kworker.0 00000100:00000001:2.0:1787828447.079093:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828447.079097:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787828447.079101:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787828447.079103:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787828447.079106:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828447.079109:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1787828447.079111:0:40701:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 02000000:00000001:1.0:1787828447.079113:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000008:00000001:2.0:1787828447.079114:0:40701:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828447.079116:0:40701:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828447.079131:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787828447.079135:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828447.079138:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828447.079138:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787828447.079140:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787828447.079143:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828447.079146:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:2.0:1787828447.079149:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787828447.079155:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2c000 x1874670646730880/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:1.0:1787828447.079158:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828447.079168:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787828447.079170:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828447.079174:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787828447.079176:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828447.079179:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1787828447.079180:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787828447.079181:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787828447.079193:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787828447.079196:0:40701:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3247 00000100:00000001:1.0:1787828447.079197:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787828447.079200:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417879808 : -126234291671808 : ffff8d30c9e2ed00) 00000100:00080000:2.0:1787828447.079200:0:40701:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787828447.079204:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828447.079207:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:1.0:1787828447.079208:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000020:00000001:2.0:1787828447.079213:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787828447.079216:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787828447.079217:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000020:00000001:2.0:1787828447.079220:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:2.0:1787828447.079222:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828447.079224:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:2.0:1787828447.079229:0:40701:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787828447.079232:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828447.079233:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787828447.079235:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828447.079237:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828447.079238:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1787828447.079241:0:40701:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000008:00000001:2.0:1787828447.079243:0:40701:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828447.079244:0:40701:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828447.079259:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787828447.079260:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828447.079261:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787828447.079262:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787828447.079263:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828447.079267:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:1.0:1787828447.079268:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00100000:2.0:1787828447.079281:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2f480 x1874670646731008/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787828447.079286:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1787828447.079288:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787828447.079292:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828447.079293:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787828447.079302:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828447.079304:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828447.079305:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787828447.079314:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828447.079320:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4297921280 00000100:00000001:1.0:1787828447.079328:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 00000100:00000001:1.0:1787828447.079336:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4297921280 00000100:00000001:1.0:1787828447.079341:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 00000100:00000001:1.0:1787828447.079345:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787828447.079348:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828447.079351:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787828447.079360:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828447/real 1787828447] req@ffff8d30c9e2ed00 x1874670646730624/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828472 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787828447.079374:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787828447.079400:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787828447.079404:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787828447.079407:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787828447.079409:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828447.079411:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787828447.079414:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787828447.079419:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2ed00 x1874670646730624/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828472 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787828447.079429:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828447.079431:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787828447.079433:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828447.079435:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787828447.079435:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828447.079437:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787828447.079439:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787828447.079456:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828447.079459:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787828447.079461:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787828447.079464:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646730624:192.168.203.143@tcp:250:kworker.0 00000100:00000001:1.0:1787828447.079468:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787828447.079469:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787828447.079470:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828447.079473:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787828447.079475:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787828447.079477:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787828447.079479:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2da40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646730752:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787828447.079482:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787828447.079484:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787828447.079485:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828447.079486:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787828447.079489:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828447.079495:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787828447.079496:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417875008 : -126234291676608 : ffff8d30c9e2da40) 00000100:00000001:1.0:1787828447.079500:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1787828447.079528:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787828447.079532:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787828447.079534:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787828447.079535:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828447.079537:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787828447.079540:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828447.079543:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4297920022 00000100:00000001:1.0:1787828447.079546:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4297921280 00000100:00000001:1.0:1787828447.079547:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4297920022 00000100:00000001:1.0:1787828447.079549:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4297921280 00000100:00000001:1.0:1787828447.079551:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787828447.079552:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828447.079554:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787828447.079557:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828447/real 1787828447] req@ffff8d30c9e2da40 x1874670646730752/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828472 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787828447.079563:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787828447.079564:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787828447.079566:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787828447.079567:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787828447.079568:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828447.079583:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787828447.079584:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787828447.079588:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2da40 x1874670646730752/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828472 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787828447.079594:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828447.079595:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787828447.079597:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828447.079599:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787828447.079650:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828447.079652:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787828447.079654:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787828447.079656:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828447.079659:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787828447.079661:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787828447.079664:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2da40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646730752:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787828447.079667:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828447.079669:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787828447.079671:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787828447.079673:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787828447.079677:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787828447.079680:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787828447.079682:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787828447.079687:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787828447.079689:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787828447.079710:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787828447.079712:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828447.079713:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787828447.079715:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787828447.079716:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787828447.079719:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787828447.079720:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787828447.079721:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787828447.079723:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787828447.079724:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787828447.079727:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787828447.079728:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828447.079731:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828447.079737:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787828447.079739:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828447.079742:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787828447.079744:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1787828447.079745:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787828447.079747:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787828447.079749:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828447.079751:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787828447.079753:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787828447.079755:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787828447.079758:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646730880:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787828447.079761:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787828447.079762:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787828447.079763:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828447.079765:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787828447.079767:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828447.079772:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787828447.079773:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417868288 : -126234291683328 : ffff8d30c9e2c000) 00000100:00000001:1.0:1787828447.079776:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1787828447.079801:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787828447.079805:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787828447.079807:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787828447.079808:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828447.079809:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787828447.079812:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828447.079814:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4297920023 00000100:00000001:1.0:1787828447.079816:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4297921280 00000100:00000001:1.0:1787828447.079818:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4297920023 00000100:00000001:1.0:1787828447.079820:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4297921280 00000100:00000001:1.0:1787828447.079821:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787828447.079823:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828447.079824:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787828447.079827:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828447/real 1787828447] req@ffff8d30c9e2c000 x1874670646730880/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828472 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787828447.079833:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787828447.079834:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787828447.079836:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787828447.079837:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787828447.079838:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828447.079839:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787828447.079840:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787828447.079843:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2c000 x1874670646730880/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828472 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787828447.079848:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828447.079849:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787828447.079868:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828447.079869:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787828447.079870:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828447.079872:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787828447.079873:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787828447.079875:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828447.079878:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787828447.079879:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787828447.079882:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646730880:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787828447.079884:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787828447.079885:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787828447.079887:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828447.079888:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787828447.079889:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787828447.079891:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787828447.079893:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646731008:192.168.203.143@tcp:8:kworker.0 00000100:00000001:1.0:1787828447.079895:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787828447.079897:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787828447.079899:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828447.079900:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787828447.079902:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828447.079907:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787828447.079908:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417881728 : -126234291669888 : ffff8d30c9e2f480) 00000100:00000001:1.0:1787828447.079912:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1787828447.079924:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787828447.079929:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787828447.079962:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787828447.079964:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828447.079965:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787828447.079968:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828447.079972:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4297920036 00000100:00000001:1.0:1787828447.079975:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4297921280 00000100:00000001:1.0:1787828447.079977:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4297920036 00000100:00000001:1.0:1787828447.079980:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4297921280 00000100:00000001:1.0:1787828447.079984:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787828447.079986:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828447.079988:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787828447.079993:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828447/real 1787828447] req@ffff8d30c9e2f480 x1874670646731008/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828472 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787828447.079999:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787828447.080000:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787828447.080002:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787828447.080003:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787828447.080004:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828447.080005:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787828447.080006:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787828447.080009:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2f480 x1874670646731008/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828472 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787828447.080014:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828447.080016:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787828447.080017:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828447.080018:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787828447.080019:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828447.080020:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787828447.080021:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787828447.080023:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828447.080025:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787828447.080026:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787828447.080028:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646731008:192.168.203.143@tcp:8:kworker.0 00000100:00000001:1.0:1787828447.080030:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828447.080031:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787828447.080032:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787828447.080033:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787828447.080035:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787828447.080036:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787828447.080038:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787828447.080039:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787828447.080041:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787828447.080042:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787828447.080043:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828447.080044:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787828447.080045:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787828447.080046:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787828447.080047:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787828447.080048:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787828447.080049:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787828447.080050:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787828447.080050:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787828447.080052:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787828447.080052:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828447.080053:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828447.080056:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787828447.080057:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828447.080058:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787828447.080059:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828447.974719:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828447.974726:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787828447.974813:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787828447.974817:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787828447.974820:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828448.998743:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828448.998767:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787828448.998851:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787828448.998854:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787828448.998857:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828450.022683:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828450.022689:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787828450.022774:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1787828450.022780:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1787828450.022784:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828451.046719:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828451.046733:0:815:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1787828451.046737:0:815:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1787828451.046742:0:815:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828451.046762:0:815:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1787828451.046766:0:815:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828451.046770:0:815:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828451.046775:0:815:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787828451.046896:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787828451.046904:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787828451.046911:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828452.070783:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828452.070795:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787828452.070849:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787828452.070856:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787828452.070861:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787828452.198771:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828452.198783:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1787828452.198786:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787828452.198791:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787828452.198797:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787828452.198805:0:40701:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 3252 00000100:00080000:0.0:1787828452.198818:0:40701:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787828452.198822:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787828452.198825:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787828452.198830:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787828452.198833:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787828452.198837:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:0.0:1787828452.198840:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:0.0:1787828452.198843:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787828452.198845:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:0.0:1787828452.198850:0:40701:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787828452.198854:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828452.198857:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787828452.198861:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787828452.198865:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828452.198868:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1787828452.198871:0:40701:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828452.198885:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787828452.198887:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828452.198891:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787828452.198894:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787828452.198899:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828452.198935:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828452.198949:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d25a40 x1874670646731136/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787828452.198963:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787828452.198968:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828452.198989:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787828452.198994:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828452.198998:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1787828452.199000:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787828452.199003:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787828452.199006:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787828452.199009:0:40701:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3252 00000100:00080000:0.0:1787828452.199014:0:40701:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787828452.199017:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787828452.199018:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787828452.199022:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787828452.199023:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787828452.199026:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000020:00000001:0.0:1787828452.199029:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:0.0:1787828452.199032:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787828452.199034:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:0.0:1787828452.199040:0:40701:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787828452.199047:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828452.199050:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787828452.199060:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787828452.199065:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828452.199068:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787828452.199070:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:1.0:1787828452.199077:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000008:00100000:0.0:1787828452.199081:0:40701:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00000001:1.0:1787828452.199083:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787828452.199088:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787828452.199095:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:0.0:1787828452.199096:0:40701:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828452.199102:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:0.0:1787828452.199105:0:40701:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828452.199109:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787828452.199116:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828452.199119:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787828452.199127:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787828452.199128:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d25a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646731136:192.168.203.143@tcp:250:kworker.0 00000100:00000001:0.0:1787828452.199138:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787828452.199148:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:1.0:1787828452.199151:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787828452.199154:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828452.199159:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:0.0:1787828452.199167:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828452.199167:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828452.199179:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00100000:0.0:1787828452.199186:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d24f00 x1874670646731264/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:1.0:1787828452.199195:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828452.199221:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787828452.199225:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828452.199230:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787828452.199232:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828452.199238:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601343040 : -126234108208576 : ffff8d30d4d25a40) 00000100:00080000:0.0:1787828452.199240:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828452.199251:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:1.0:1787828452.199253:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:0.0:1787828452.199255:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787828452.199263:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787828452.199279:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787828452.199285:0:40701:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3252 00000100:00080000:0.0:1787828452.199321:0:40701:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787828452.199326:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787828452.199329:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787828452.199334:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787828452.199338:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787828452.199345:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000800:00000100:1.0:1787828452.199348:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000020:00000001:0.0:1787828452.199357:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:0.0:1787828452.199372:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787828452.199376:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:1.0:1787828452.199384:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00080000:0.0:1787828452.199385:0:40701:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787828452.199394:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828452.199398:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1787828452.199400:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 02000000:00000001:0.0:1787828452.199404:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787828452.199408:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828452.199410:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828452.199419:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:1.0:1787828452.199419:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000008:00100000:0.0:1787828452.199428:0:40701:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1787828452.199433:0:40701:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828452.199436:0:40701:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828452.199438:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828452.199443:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:1.0:1787828452.199447:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4297926400 02000000:00000001:0.0:1787828452.199449:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828452.199459:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787828452.199462:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787828452.199464:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828452.199479:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1787828452.199482:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828452.199492:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4297926400 00000100:00100000:0.0:1787828452.199494:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d24000 x1874670646731392/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787828452.199505:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1787828452.199516:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787828452.199519:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828452.199523:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787828452.199528:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1787828452.199528:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787828452.199538:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787828452.199544:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1787828452.199545:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00000001:0.0:1787828452.199554:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00100000:1.0:1787828452.199560:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828452/real 1787828452] req@ffff8d30d4d25a40 x1874670646731136/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828477 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828452.199562:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787828452.199576:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787828452.199588:0:40701:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3252 00000400:00000001:1.0:1787828452.199589:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787828452.199594:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00080000:0.0:1787828452.199668:0:40701:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787828452.199675:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787828452.199678:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787828452.199679:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:1.0:1787828452.199687:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000020:00000001:0.0:1787828452.199688:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787828452.199694:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:1.0:1787828452.199695:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1787828452.199698:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:1.0:1787828452.199701:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000020:00000001:0.0:1787828452.199707:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:1.0:1787828452.199708:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828452.199719:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787828452.199725:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00100000:1.0:1787828452.199727:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d25a40 x1874670646731136/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828477 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828452.199735:0:40701:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787828452.199745:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828452.199754:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787828452.199762:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787828452.199771:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828452.199780:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:1.0:1787828452.199783:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000008:00100000:0.0:1787828452.199786:0:40701:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00000001:1.0:1787828452.199791:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000008:00000001:0.0:1787828452.199793:0:40701:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828452.199802:0:40701:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828452.199803:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828452.199809:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787828452.199813:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:0.0:1787828452.199814:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787828452.199821:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828452.199822:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787828452.199830:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787828452.199831:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787828452.199836:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00080000:1.0:1787828452.199839:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 02000000:00000001:0.0:1787828452.199841:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1787828452.199851:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787828452.199853:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828452.199856:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787828452.199871:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d25a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646731136:192.168.203.143@tcp:250:kworker.0 00000100:00100000:0.0:1787828452.199872:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d270c0 x1874670646731520/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787828452.199883:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828452.199892:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787828452.199896:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 00000100:00000001:0.0:1787828452.199899:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 02000000:00000001:1.0:1787828452.199902:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 00000100:00000001:0.0:1787828452.199932:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828452.199939:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828452.199941:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787828452.199951:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787828452.199958:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787828452.199964:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787828452.199966:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787828452.199972:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787828452.199974:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828452.199977:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828452.199987:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787828452.199991:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828452.199995:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787828452.199998:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1787828452.200001:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787828452.200003:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787828452.200006:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828452.200010:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787828452.200013:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787828452.200017:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787828452.200023:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d24f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646731264:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787828452.200028:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787828452.200031:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787828452.200034:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828452.200036:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787828452.200040:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828452.200049:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787828452.200050:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601340160 : -126234108211456 : ffff8d30d4d24f00) 00000100:00000001:1.0:1787828452.200056:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1787828452.200076:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787828452.200082:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787828452.200085:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787828452.200087:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828452.200089:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787828452.200093:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828452.200098:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4297925143 00000100:00000001:1.0:1787828452.200102:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4297926400 00000100:00000001:1.0:1787828452.200105:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4297925143 00000100:00000001:1.0:1787828452.200108:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4297926400 00000100:00000001:1.0:1787828452.200110:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787828452.200113:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828452.200115:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787828452.200121:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828452/real 1787828452] req@ffff8d30d4d24f00 x1874670646731264/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828477 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787828452.200131:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787828452.200132:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787828452.200136:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787828452.200138:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787828452.200140:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828452.200142:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787828452.200144:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787828452.200148:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d24f00 x1874670646731264/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828477 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787828452.200177:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828452.200182:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787828452.200186:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828452.200189:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787828452.200192:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828452.200196:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787828452.200199:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787828452.200205:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828452.200211:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787828452.200216:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787828452.200223:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d24f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646731264:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787828452.200237:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787828452.200240:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787828452.200243:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828452.200247:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787828452.200250:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787828452.200255:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787828452.200262:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d24000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646731392:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787828452.200268:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787828452.200272:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787828452.200275:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828452.200278:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787828452.200283:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828452.200299:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787828452.200301:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601336320 : -126234108215296 : ffff8d30d4d24000) 00000100:00000001:1.0:1787828452.200305:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1787828452.200325:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787828452.200334:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787828452.200339:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787828452.200342:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828452.200346:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787828452.200351:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828452.200358:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4297925143 00000100:00000001:1.0:1787828452.200374:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4297926400 00000100:00000001:1.0:1787828452.200379:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4297925143 00000100:00000001:1.0:1787828452.200383:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4297926400 00000100:00000001:1.0:1787828452.200387:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787828452.200392:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828452.200396:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787828452.200405:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828452/real 1787828452] req@ffff8d30d4d24000 x1874670646731392/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828477 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787828452.200422:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787828452.200424:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787828452.200429:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787828452.200432:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787828452.200433:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828452.200436:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787828452.200438:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787828452.200443:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d24000 x1874670646731392/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828477 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787828452.200452:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828452.200455:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787828452.200458:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828452.200460:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787828452.200470:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828452.200472:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787828452.200474:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787828452.200477:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828452.200480:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787828452.200483:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787828452.200487:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d24000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646731392:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787828452.200492:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787828452.200494:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787828452.200496:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828452.200499:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787828452.200501:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787828452.200503:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787828452.200507:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d270c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646731520:192.168.203.143@tcp:8:kworker.0 00000100:00000001:1.0:1787828452.200511:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787828452.200514:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787828452.200516:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828452.200518:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787828452.200522:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828452.200529:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787828452.200530:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601348800 : -126234108202816 : ffff8d30d4d270c0) 00000100:00000001:1.0:1787828452.200534:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1787828452.200550:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787828452.200555:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787828452.200558:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787828452.200560:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828452.200562:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787828452.200565:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828452.200569:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4297925156 00000100:00000001:1.0:1787828452.200572:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4297926400 00000100:00000001:1.0:1787828452.200575:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4297925156 00000100:00000001:1.0:1787828452.200578:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4297926400 00000100:00000001:1.0:1787828452.200580:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787828452.200583:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828452.200585:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787828452.200589:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828452/real 1787828452] req@ffff8d30d4d270c0 x1874670646731520/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828477 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787828452.200638:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787828452.200639:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787828452.200643:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787828452.200645:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787828452.200647:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828452.200649:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787828452.200651:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787828452.200656:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d270c0 x1874670646731520/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828477 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787828452.200665:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828452.200668:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787828452.200671:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828452.200673:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787828452.200674:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828452.200676:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787828452.200679:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787828452.200681:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828452.200685:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787828452.200687:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787828452.200691:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d270c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646731520:192.168.203.143@tcp:8:kworker.0 00000100:00000001:1.0:1787828452.200695:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828452.200698:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787828452.200700:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787828452.200702:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787828452.200705:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787828452.200707:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787828452.200709:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787828452.200712:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787828452.200713:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787828452.200718:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787828452.200719:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828452.200722:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787828452.200724:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787828452.200725:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787828452.200729:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787828452.200730:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787828452.200732:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787828452.200734:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787828452.200735:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787828452.200737:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787828452.200738:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828452.200740:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787828452.200742:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787828452.200743:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787828452.200745:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787828452.200746:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787828452.200747:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787828452.200749:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787828452.200750:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787828452.200753:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787828452.200755:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828452.200757:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828452.200761:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787828452.200763:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828452.200765:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787828452.200767:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828453.094729:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828453.094738:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787828453.094880:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1787828453.094886:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1787828453.094891:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828454.118739:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828454.118747:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787828454.118833:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787828454.118839:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787828454.118844:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828455.142791:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828455.142802:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787828455.142945:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787828455.142951:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787828455.142956:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828456.166657:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828456.166664:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787828456.166749:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1787828456.166755:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1787828456.166759:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828457.190687:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828457.190694:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787828457.190792:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787828457.190797:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787828457.190801:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787828457.318718:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828457.318730:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1787828457.318733:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787828457.318738:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787828457.318743:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787828457.318752:0:468:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 3257 00000100:00080000:3.0:1787828457.318764:0:468:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787828457.318768:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787828457.318770:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:3.0:1787828457.318775:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1787828457.318778:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1787828457.318782:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:3.0:1787828457.318785:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:3.0:1787828457.318788:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787828457.318789:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:3.0:1787828457.318795:0:468:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:3.0:1787828457.318798:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787828457.318802:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787828457.318805:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787828457.318808:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828457.318811:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:3.0:1787828457.318814:0:468:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787828457.318826:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787828457.318827:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828457.318831:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787828457.318834:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787828457.318839:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828457.318853:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787828457.318863:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c70143c0 x1874670646731648/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787828457.318899:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787828457.318906:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828457.318943:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787828457.318947:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828457.318952:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1787828457.318954:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787828457.318956:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787828457.318960:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787828457.318963:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3257 00000100:00080000:3.0:1787828457.318969:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787828457.318972:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787828457.318974:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:3.0:1787828457.318978:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1787828457.318980:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1787828457.318982:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000020:00000001:3.0:1787828457.318985:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:3.0:1787828457.318988:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787828457.318991:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:3.0:1787828457.318995:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:3.0:1787828457.318999:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787828457.319001:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787828457.319009:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787828457.319013:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:3.0:1787828457.319013:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828457.319021:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:1.0:1787828457.319024:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000008:00100000:3.0:1787828457.319027:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00000001:1.0:1787828457.319030:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000008:00000001:3.0:1787828457.319035:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828457.319037:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787828457.319042:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828457.319044:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787828457.319051:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787828457.319056:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828457.319059:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787828457.319061:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787828457.319063:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828457.319064:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787828457.319071:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787828457.319076:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828457.319081:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787828457.319085:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c70143c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646731648:192.168.203.143@tcp:250:kworker.0 00000100:00100000:3.0:1787828457.319096:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c7015680 x1874670646731776/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787828457.319098:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787828457.319107:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787828457.319114:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828457.319115:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 02000000:00000001:1.0:1787828457.319123:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:3.0:1787828457.319125:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828457.319132:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828457.319133:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787828457.319136:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828457.319145:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1787828457.319148:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787828457.319151:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787828457.319154:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787828457.319158:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3257 00000100:00000001:1.0:1787828457.319159:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00080000:3.0:1787828457.319162:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1787828457.319165:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369536448 : -126234340015168 : ffff8d30c70143c0) 00000100:00000001:3.0:1787828457.319168:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787828457.319175:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:1.0:1787828457.319177:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000020:00000001:3.0:1787828457.319182:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1787828457.319186:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1787828457.319189:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000020:00000001:3.0:1787828457.319192:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:3.0:1787828457.319195:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787828457.319197:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:3.0:1787828457.319201:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:3.0:1787828457.319204:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787828457.319206:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787828457.319210:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787828457.319213:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828457.319216:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:3.0:1787828457.319218:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1787828457.319221:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828457.319223:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:1.0:1787828457.319231:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 02000000:00000001:3.0:1787828457.319233:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787828457.319238:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828457.319241:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787828457.319243:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:1.0:1787828457.319247:0:2390:0:(events.c:34:request_out_callback()) Process entered 02000000:00000001:3.0:1787828457.319248:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828457.319259:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787828457.319260:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828457.319267:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787828457.319274:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c70152c0 x1874670646731904/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787828457.319285:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1787828457.319297:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787828457.319298:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828457.319321:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828457.319326:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4297931520 00000100:00000001:3.0:1787828457.319331:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787828457.319339:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1787828457.319340:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 00000100:00080000:3.0:1787828457.319350:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1787828457.319356:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787828457.319359:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787828457.319363:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4297931520 00000100:00000001:3.0:1787828457.319365:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787828457.319376:0:468:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3257 00000100:00000001:1.0:1787828457.319379:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 00000100:00080000:3.0:1787828457.319386:0:468:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1787828457.319387:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787828457.319398:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828457.319398:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787828457.319406:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:1.0:1787828457.319407:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000020:00000001:3.0:1787828457.319416:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1787828457.319423:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00100000:1.0:1787828457.319424:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828457/real 1787828457] req@ffff8d30c70143c0 x1874670646731648/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828482 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:3.0:1787828457.319429:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000020:00000001:3.0:1787828457.319440:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:3.0:1787828457.319452:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000001:1.0:1787828457.319453:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000001:3.0:1787828457.319456:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000400:00000001:1.0:1787828457.319459:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787828457.319467:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00080000:3.0:1787828457.319472:0:468:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:1.0:1787828457.319474:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787828457.319480:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787828457.319481:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828457.319489:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 02000000:00000001:3.0:1787828457.319489:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1787828457.319496:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:3.0:1787828457.319498:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787828457.319503:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828457.319505:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:3.0:1787828457.319508:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1787828457.319512:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828457.319517:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787828457.319518:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c70143c0 x1874670646731648/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828482 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1787828457.319523:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787828457.319528:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828457.319536:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787828457.319540:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787828457.319545:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828457.319557:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828457.319559:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828457.319567:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00100000:3.0:1787828457.319569:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c7014b40 x1874670646732032/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787828457.319573:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828457.319579:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787828457.319583:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828457.319590:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787828457.319594:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787828457.319596:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787828457.319599:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828457.319603:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1787828457.319645:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828457.319652:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787828457.319655:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787828457.319660:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c70143c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646731648:192.168.203.143@tcp:250:kworker.0 00000100:00000001:1.0:1787828457.319669:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828457.319672:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787828457.319674:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787828457.319677:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787828457.319682:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787828457.319684:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787828457.319687:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787828457.319691:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787828457.319693:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787828457.319696:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787828457.319697:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828457.319700:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828457.319708:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787828457.319710:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828457.319713:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787828457.319715:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1787828457.319717:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787828457.319719:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787828457.319721:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828457.319724:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787828457.319727:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787828457.319729:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787828457.319734:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c7015680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646731776:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787828457.319738:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787828457.319740:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787828457.319742:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828457.319745:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787828457.319748:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828457.319755:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787828457.319757:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369541248 : -126234340010368 : ffff8d30c7015680) 00000100:00000001:1.0:1787828457.319763:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1787828457.319780:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787828457.319785:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787828457.319788:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787828457.319790:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828457.319792:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787828457.319796:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828457.319801:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4297930263 00000100:00000001:1.0:1787828457.319804:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4297931520 00000100:00000001:1.0:1787828457.319807:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4297930263 00000100:00000001:1.0:1787828457.319810:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4297931520 00000100:00000001:1.0:1787828457.319812:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787828457.319815:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828457.319817:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787828457.319822:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828457/real 1787828457] req@ffff8d30c7015680 x1874670646731776/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828482 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787828457.319832:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787828457.319834:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787828457.319837:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787828457.319838:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787828457.319840:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828457.319842:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787828457.319844:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787828457.319848:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c7015680 x1874670646731776/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828482 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787828457.319865:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828457.319867:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787828457.319891:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828457.319893:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787828457.319895:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828457.319916:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787828457.319918:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787828457.319922:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828457.319926:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787828457.319929:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787828457.319933:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c7015680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646731776:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787828457.319939:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787828457.319941:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787828457.319943:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828457.319945:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787828457.319947:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787828457.319950:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787828457.319953:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c70152c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646731904:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787828457.319957:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787828457.319960:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787828457.319962:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828457.319964:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787828457.319969:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828457.319975:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787828457.319977:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369540288 : -126234340011328 : ffff8d30c70152c0) 00000100:00000001:1.0:1787828457.319981:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1787828457.319997:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787828457.320002:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787828457.320006:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787828457.320008:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828457.320010:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787828457.320013:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828457.320018:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4297930263 00000100:00000001:1.0:1787828457.320021:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4297931520 00000100:00000001:1.0:1787828457.320024:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4297930263 00000100:00000001:1.0:1787828457.320026:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4297931520 00000100:00000001:1.0:1787828457.320029:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787828457.320031:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828457.320034:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787828457.320038:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828457/real 1787828457] req@ffff8d30c70152c0 x1874670646731904/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828482 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787828457.320048:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787828457.320050:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787828457.320052:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787828457.320054:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787828457.320055:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828457.320058:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787828457.320059:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787828457.320064:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c70152c0 x1874670646731904/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828482 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787828457.320073:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828457.320076:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787828457.320079:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828457.320081:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787828457.320082:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828457.320084:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787828457.320086:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787828457.320089:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828457.320093:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787828457.320095:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787828457.320100:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c70152c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646731904:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787828457.320104:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787828457.320106:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787828457.320108:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828457.320111:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787828457.320113:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787828457.320115:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787828457.320118:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c7014b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646732032:192.168.203.143@tcp:8:kworker.0 00000100:00000001:1.0:1787828457.320122:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787828457.320124:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787828457.320126:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828457.320128:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787828457.320132:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828457.320138:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787828457.320140:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369538368 : -126234340013248 : ffff8d30c7014b40) 00000100:00000001:1.0:1787828457.320143:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1787828457.320154:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787828457.320159:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787828457.320162:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787828457.320163:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828457.320166:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787828457.320169:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828457.320172:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4297930276 00000100:00000001:1.0:1787828457.320175:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4297931520 00000100:00000001:1.0:1787828457.320178:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4297930276 00000100:00000001:1.0:1787828457.320181:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4297931520 00000100:00000001:1.0:1787828457.320183:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787828457.320186:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828457.320188:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787828457.320192:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828457/real 1787828457] req@ffff8d30c7014b40 x1874670646732032/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828482 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787828457.320209:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787828457.320211:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787828457.320213:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787828457.320215:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787828457.320217:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828457.320219:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787828457.320221:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787828457.320225:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c7014b40 x1874670646732032/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828482 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787828457.320233:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828457.320235:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787828457.320238:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828457.320240:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787828457.320241:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828457.320243:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787828457.320245:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787828457.320248:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828457.320252:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787828457.320254:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787828457.320258:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c7014b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646732032:192.168.203.143@tcp:8:kworker.0 00000100:00000001:1.0:1787828457.320263:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828457.320265:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787828457.320267:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787828457.320269:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787828457.320272:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787828457.320274:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787828457.320275:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787828457.320278:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787828457.320279:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787828457.320282:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787828457.320284:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828457.320286:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787828457.320288:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787828457.320290:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787828457.320293:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787828457.320295:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787828457.320296:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787828457.320298:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787828457.320300:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787828457.320301:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787828457.320303:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828457.320305:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787828457.320306:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787828457.320325:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787828457.320327:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787828457.320328:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787828457.320330:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787828457.320331:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787828457.320333:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787828457.320334:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787828457.320336:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828457.320338:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828457.320341:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787828457.320343:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828457.320345:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787828457.320347:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828458.214794:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828458.214803:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787828458.214932:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787828458.214940:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787828458.214947:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828459.238740:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828459.238759:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787828459.238876:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1787828459.238911:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1787828459.238915:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828460.262739:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828460.262744:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787828460.262868:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787828460.262872:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787828460.262877:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828461.286708:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828461.286715:0:815:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1787828461.286718:0:815:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1787828461.286722:0:815:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828461.286727:0:815:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1787828461.286729:0:815:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828461.286732:0:815:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828461.286734:0:815:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787828461.286782:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787828461.286786:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787828461.286788:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828462.310672:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828462.310677:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787828462.310732:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1787828462.310735:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1787828462.310738:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787828462.438688:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828462.438696:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1787828462.438698:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787828462.438701:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787828462.438704:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787828462.438709:0:40701:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 3262 00000100:00080000:2.0:1787828462.438717:0:40701:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787828462.438719:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828462.438721:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787828462.438724:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787828462.438726:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787828462.438728:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:2.0:1787828462.438730:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:2.0:1787828462.438731:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828462.438732:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:2.0:1787828462.438736:0:40701:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787828462.438738:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828462.438740:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787828462.438743:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828462.438745:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828462.438746:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:2.0:1787828462.438748:0:40701:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828462.438758:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787828462.438759:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828462.438761:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787828462.438763:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787828462.438767:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828462.438776:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787828462.438783:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2c3c0 x1874670646732160/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787828462.438791:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787828462.438794:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828462.438805:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787828462.438807:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828462.438810:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1787828462.438811:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787828462.438812:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787828462.438814:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787828462.438816:0:40701:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3262 00000100:00080000:2.0:1787828462.438819:0:40701:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787828462.438821:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828462.438822:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787828462.438824:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787828462.438826:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:1.0:1787828462.438827:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:2.0:1787828462.438828:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000020:00000001:2.0:1787828462.438832:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:1.0:1787828462.438833:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1787828462.438834:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787828462.438845:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787828462.438847:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787828462.438850:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828462.438853:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 02000000:00000001:1.0:1787828462.438855:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000100:00080000:2.0:1787828462.438859:0:40701:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 02000000:00000001:1.0:1787828462.438860:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787828462.438863:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828462.438864:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828462.438866:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00100000:1.0:1787828462.438870:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646732160:192.168.203.143@tcp:250:kworker.0 02000000:00000001:2.0:1787828462.438871:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828462.438877:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828462.438878:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:2.0:1787828462.438880:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 02000000:00000001:1.0:1787828462.438883:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000008:00100000:2.0:1787828462.438884:0:40701:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 02000000:00000001:1.0:1787828462.438887:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:2.0:1787828462.438888:0:40701:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828462.438894:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:2.0:1787828462.438894:0:40701:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828462.438900:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828462.438901:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787828462.438905:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828462.438906:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787828462.438907:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787828462.438908:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828462.438914:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828462.438916:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787828462.438920:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417869248 : -126234291682368 : ffff8d30c9e2c3c0) 00000100:00100000:2.0:1787828462.438920:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2f0c0 x1874670646732288/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787828462.438929:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:2.0:1787828462.438933:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787828462.438935:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828462.438937:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787828462.438939:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828462.438941:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1787828462.438942:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787828462.438944:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787828462.438946:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787828462.438948:0:40701:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3262 00000100:00080000:2.0:1787828462.438951:0:40701:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787828462.438952:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828462.438953:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787828462.438954:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787828462.438956:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787828462.438958:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000020:00000001:2.0:1787828462.438960:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:2.0:1787828462.438961:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828462.438962:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:2.0:1787828462.438964:0:40701:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787828462.438965:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828462.438966:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787828462.438969:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000800:00000100:1.0:1787828462.438973:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 02000000:00000001:2.0:1787828462.438973:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828462.438977:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1787828462.438978:0:40701:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1787828462.438980:0:40701:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828462.438982:0:40701:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828462.438985:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:1.0:1787828462.438986:0:2390:0:(events.c:34:request_out_callback()) Process entered 02000000:00000001:2.0:1787828462.438987:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828462.438990:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787828462.438992:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:1.0:1787828462.438993:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 02000000:00000001:2.0:1787828462.438994:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828462.438997:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828462.438999:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828462.439000:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00100000:2.0:1787828462.439007:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2e940 x1874670646732416/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787828462.439008:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828462.439014:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4297936640 00000100:00000001:2.0:1787828462.439017:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787828462.439019:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828462.439021:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787828462.439023:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1787828462.439026:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 00000100:00080000:2.0:1787828462.439026:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1787828462.439029:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787828462.439032:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787828462.439034:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:1.0:1787828462.439037:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4297936640 00000100:00080000:2.0:1787828462.439037:0:40701:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3262 00000100:00080000:2.0:1787828462.439041:0:40701:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787828462.439045:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787828462.439046:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 00000100:00000001:2.0:1787828462.439046:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787828462.439051:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:1.0:1787828462.439053:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000020:00000001:2.0:1787828462.439053:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787828462.439054:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000020:00000001:2.0:1787828462.439060:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:1.0:1787828462.439061:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828462.439062:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828462.439065:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:1.0:1787828462.439067:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00080000:2.0:1787828462.439069:0:40701:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787828462.439072:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828462.439075:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00100000:1.0:1787828462.439076:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828462/real 1787828462] req@ffff8d30c9e2c3c0 x1874670646732160/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828487 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1787828462.439078:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828462.439084:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828462.439088:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1787828462.439090:0:40701:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000008:00000001:2.0:1787828462.439095:0:40701:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828462.439097:0:40701:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:1.0:1787828462.439100:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 02000000:00000001:2.0:1787828462.439101:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000400:00000001:1.0:1787828462.439105:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 02000000:00000001:2.0:1787828462.439105:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828462.439109:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787828462.439110:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787828462.439112:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828462.439113:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1787828462.439115:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828462.439118:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00100000:2.0:1787828462.439122:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2c780 x1874670646732544/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787828462.439123:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828462.439129:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787828462.439134:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828462.439134:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787828462.439138:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828462.439141:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787828462.439144:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2c3c0 x1874670646732160/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828487 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787828462.439156:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828462.439160:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787828462.439162:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828462.439164:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787828462.439164:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828462.439166:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787828462.439167:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787828462.439169:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828462.439172:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787828462.439174:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787828462.439176:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646732160:192.168.203.143@tcp:250:kworker.0 00000100:00000001:1.0:1787828462.439180:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828462.439181:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787828462.439182:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787828462.439184:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787828462.439187:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787828462.439188:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787828462.439190:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787828462.439193:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787828462.439194:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787828462.439196:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787828462.439197:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828462.439199:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828462.439205:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787828462.439206:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828462.439208:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787828462.439209:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1787828462.439210:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787828462.439211:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787828462.439212:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828462.439214:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787828462.439215:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787828462.439217:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787828462.439219:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646732288:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787828462.439221:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787828462.439223:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787828462.439224:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828462.439226:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787828462.439228:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828462.439232:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787828462.439233:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417880768 : -126234291670848 : ffff8d30c9e2f0c0) 00000100:00000001:1.0:1787828462.439237:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1787828462.439255:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787828462.439258:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787828462.439260:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787828462.439261:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828462.439262:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787828462.439264:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828462.439267:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4297935382 00000100:00000001:1.0:1787828462.439269:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4297936640 00000100:00000001:1.0:1787828462.439271:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4297935382 00000100:00000001:1.0:1787828462.439272:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4297936640 00000100:00000001:1.0:1787828462.439274:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787828462.439275:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828462.439276:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787828462.439279:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828462/real 1787828462] req@ffff8d30c9e2f0c0 x1874670646732288/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828487 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787828462.439284:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787828462.439285:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787828462.439286:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787828462.439294:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787828462.439295:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828462.439297:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787828462.439299:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787828462.439307:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2f0c0 x1874670646732288/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828487 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787828462.439320:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828462.439323:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787828462.439326:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828462.439327:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787828462.439328:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828462.439331:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787828462.439332:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787828462.439335:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828462.439338:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787828462.439340:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787828462.439343:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646732288:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787828462.439347:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787828462.439348:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787828462.439350:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828462.439352:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787828462.439353:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787828462.439356:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787828462.439359:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646732416:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787828462.439363:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787828462.439365:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787828462.439366:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828462.439369:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787828462.439372:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828462.439377:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787828462.439379:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417878848 : -126234291672768 : ffff8d30c9e2e940) 00000100:00000001:1.0:1787828462.439382:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1787828462.439395:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787828462.439399:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787828462.439400:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787828462.439402:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828462.439403:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787828462.439405:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828462.439408:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4297935382 00000100:00000001:1.0:1787828462.439410:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4297936640 00000100:00000001:1.0:1787828462.439411:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4297935382 00000100:00000001:1.0:1787828462.439413:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4297936640 00000100:00000001:1.0:1787828462.439414:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787828462.439416:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828462.439417:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787828462.439420:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828462/real 1787828462] req@ffff8d30c9e2e940 x1874670646732416/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828487 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787828462.439425:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787828462.439425:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787828462.439427:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787828462.439428:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787828462.439429:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828462.439430:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787828462.439431:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787828462.439433:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2e940 x1874670646732416/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828487 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787828462.439438:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828462.439439:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787828462.439441:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828462.439442:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787828462.439443:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828462.439444:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787828462.439445:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787828462.439446:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828462.439448:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787828462.439450:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787828462.439452:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646732416:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787828462.439455:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787828462.439456:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787828462.439457:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828462.439458:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787828462.439459:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787828462.439460:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787828462.439462:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646732544:192.168.203.143@tcp:8:kworker.0 00000100:00000001:1.0:1787828462.439464:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787828462.439466:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787828462.439467:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828462.439468:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787828462.439470:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828462.439473:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787828462.439474:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417870208 : -126234291681408 : ffff8d30c9e2c780) 00000100:00000001:1.0:1787828462.439476:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1787828462.439483:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787828462.439486:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787828462.439487:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787828462.439488:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828462.439489:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787828462.439490:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828462.439493:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4297935395 00000100:00000001:1.0:1787828462.439494:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4297936640 00000100:00000001:1.0:1787828462.439496:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4297935395 00000100:00000001:1.0:1787828462.439497:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4297936640 00000100:00000001:1.0:1787828462.439499:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787828462.439500:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828462.439501:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787828462.439503:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828462/real 1787828462] req@ffff8d30c9e2c780 x1874670646732544/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828487 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787828462.439512:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787828462.439513:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787828462.439515:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787828462.439516:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787828462.439516:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828462.439517:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787828462.439518:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787828462.439520:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2c780 x1874670646732544/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828487 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787828462.439524:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828462.439526:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787828462.439527:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828462.439528:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787828462.439529:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828462.439530:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787828462.439531:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787828462.439532:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828462.439534:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787828462.439535:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787828462.439537:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646732544:192.168.203.143@tcp:8:kworker.0 00000100:00000001:1.0:1787828462.439539:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828462.439540:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787828462.439541:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787828462.439542:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787828462.439543:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787828462.439544:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787828462.439545:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787828462.439546:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787828462.439547:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787828462.439550:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787828462.439551:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828462.439552:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787828462.439553:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787828462.439553:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787828462.439555:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787828462.439556:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787828462.439556:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787828462.439558:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787828462.439558:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787828462.439561:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787828462.439561:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828462.439562:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787828462.439563:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787828462.439564:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787828462.439565:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787828462.439565:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787828462.439566:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787828462.439567:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787828462.439567:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787828462.439568:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787828462.439569:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828462.439570:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828462.439572:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787828462.439573:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828462.439575:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787828462.439576:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828463.334800:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828463.334807:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787828463.334933:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787828463.334942:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787828463.334949:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828464.358673:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828464.358678:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787828464.358795:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787828464.358804:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787828464.358810:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828465.382768:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828465.382777:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787828465.382855:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1787828465.382864:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1787828465.382870:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828466.406792:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828466.406803:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787828466.406880:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787828466.406885:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787828466.406890:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828467.430778:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828467.430839:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787828467.430998:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787828467.431007:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787828467.431013:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787828467.558741:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828467.558763:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1787828467.558766:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787828467.558771:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787828467.558776:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787828467.558785:0:40701:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 3268 00000100:00080000:2.0:1787828467.558813:0:40701:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787828467.558818:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828467.558821:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787828467.558827:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787828467.558831:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787828467.558850:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:2.0:1787828467.558864:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:2.0:1787828467.558867:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828467.558869:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:2.0:1787828467.558875:0:40701:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787828467.558879:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828467.558884:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787828467.558889:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828467.558892:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828467.558894:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:2.0:1787828467.558905:0:40701:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828467.558917:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787828467.558918:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828467.558922:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787828467.558925:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787828467.558929:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828467.558941:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787828467.558952:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2de00 x1874670646732672/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787828467.558962:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787828467.558966:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828467.558987:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787828467.558990:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828467.558994:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1787828467.558996:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787828467.558999:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787828467.559003:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787828467.559006:0:40701:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3268 00000100:00080000:2.0:1787828467.559011:0:40701:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787828467.559013:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828467.559015:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787828467.559018:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787828467.559020:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787828467.559022:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000020:00000001:2.0:1787828467.559025:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:2.0:1787828467.559027:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828467.559030:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:2.0:1787828467.559034:0:40701:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787828467.559036:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828467.559038:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787828467.559045:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828467.559047:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828467.559051:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1787828467.559057:0:40701:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1787828467.559060:0:40701:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828467.559062:0:40701:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828467.559067:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787828467.559068:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828467.559070:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787828467.559071:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787828467.559074:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787828467.559076:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828467.559080:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787828467.559085:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2fc00 x1874670646732800/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787828467.559108:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787828467.559110:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828467.559112:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787828467.559115:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828467.559120:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1787828467.559121:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787828467.559123:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787828467.559126:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787828467.559130:0:40701:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3268 00000100:00080000:2.0:1787828467.559135:0:40701:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787828467.559138:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828467.559139:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787828467.559141:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787828467.559143:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787828467.559144:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000020:00000001:2.0:1787828467.559145:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:2.0:1787828467.559147:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828467.559148:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:2.0:1787828467.559152:0:40701:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787828467.559156:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828467.559157:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 02000000:00000001:2.0:1787828467.559158:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:3.0:1787828467.559161:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 02000000:00000001:2.0:1787828467.559163:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787828467.559165:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 02000000:00000001:2.0:1787828467.559166:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828467.559170:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1787828467.559174:0:40701:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1787828467.559179:0:40701:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828467.559180:0:40701:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828467.559185:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787828467.559187:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828467.559188:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787828467.559189:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787828467.559191:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828467.559194:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787828467.559197:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2d2c0 x1874670646732928/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787828467.559202:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787828467.559203:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828467.559206:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787828467.559209:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828467.559212:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1787828467.559213:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787828467.559214:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787828467.559216:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787828467.559218:0:40701:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3268 00000100:00080000:2.0:1787828467.559220:0:40701:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787828467.559221:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828467.559222:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787828467.559223:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787828467.559224:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787828467.559226:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000020:00000001:2.0:1787828467.559227:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:2.0:1787828467.559229:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828467.559230:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:2.0:1787828467.559231:0:40701:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787828467.559232:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828467.559234:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787828467.559236:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828467.559237:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828467.559239:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1787828467.559240:0:40701:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000008:00000001:2.0:1787828467.559242:0:40701:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828467.559249:0:40701:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828467.559253:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787828467.559254:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828467.559255:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787828467.559256:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787828467.559257:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828467.559260:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787828467.559263:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2cb40 x1874670646733056/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787828467.559267:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787828467.559269:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828467.559270:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828467.559319:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787828467.559326:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787828467.559329:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787828467.559332:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787828467.559341:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646732672:192.168.203.143@tcp:250:kworker.0 00000100:00000001:3.0:1787828467.559346:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1787828467.559386:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1787828467.559388:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787828467.559391:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1787828467.559396:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828467.559413:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787828467.559415:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417875968 : -126234291675648 : ffff8d30c9e2de00) 00000100:00000001:3.0:1787828467.559419:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1787828467.559453:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:3.0:1787828467.559462:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787828467.559466:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787828467.559467:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828467.559469:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1787828467.559472:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828467.559476:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4297941760 00000100:00000001:3.0:1787828467.559483:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 00000100:00000001:3.0:1787828467.559489:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4297941760 00000100:00000001:3.0:1787828467.559492:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 00000100:00000001:3.0:1787828467.559494:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787828467.559496:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828467.559498:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1787828467.559504:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828467/real 1787828467] req@ffff8d30c9e2de00 x1874670646732672/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828492 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1787828467.559514:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787828467.559515:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787828467.559518:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787828467.559519:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787828467.559521:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787828467.559522:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787828467.559524:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787828467.559527:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2de00 x1874670646732672/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828492 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787828467.559533:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787828467.559535:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787828467.559538:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787828467.559539:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787828467.559540:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787828467.559542:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787828467.559544:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787828467.559546:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828467.559549:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787828467.559551:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787828467.559553:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646732672:192.168.203.143@tcp:250:kworker.0 00000100:00000001:3.0:1787828467.559557:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787828467.559558:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787828467.559559:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787828467.559560:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787828467.559562:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787828467.559564:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787828467.559566:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646732800:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787828467.559568:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1787828467.559570:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1787828467.559571:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787828467.559573:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1787828467.559575:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828467.559578:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787828467.559597:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417883648 : -126234291667968 : ffff8d30c9e2fc00) 00000100:00000001:3.0:1787828467.559602:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1787828467.559676:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:3.0:1787828467.559686:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787828467.559690:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787828467.559692:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828467.559695:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1787828467.559703:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828467.559709:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4297940503 00000100:00000001:3.0:1787828467.559731:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4297941760 00000100:00000001:3.0:1787828467.559735:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4297940503 00000100:00000001:3.0:1787828467.559738:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4297941760 00000100:00000001:3.0:1787828467.559741:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787828467.559745:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828467.559748:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1787828467.559754:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828467/real 1787828467] req@ffff8d30c9e2fc00 x1874670646732800/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828492 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1787828467.559765:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787828467.559766:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787828467.559770:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787828467.559772:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787828467.559774:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787828467.559776:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787828467.559778:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787828467.559782:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2fc00 x1874670646732800/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828492 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787828467.559791:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787828467.559793:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787828467.559797:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787828467.559826:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787828467.559828:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787828467.559830:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787828467.559833:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787828467.559865:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828467.559868:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787828467.559869:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787828467.559872:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646732800:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787828467.559875:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787828467.559876:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787828467.559878:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787828467.559880:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787828467.559882:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787828467.559884:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787828467.559886:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787828467.559889:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787828467.559890:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787828467.559895:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787828467.559896:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787828467.559897:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787828467.559898:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787828467.559899:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787828467.559901:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787828467.559901:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787828467.559902:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787828467.559903:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787828467.559904:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787828467.559906:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787828467.559907:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787828467.559908:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787828467.559915:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787828467.559916:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828467.559918:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1787828467.559919:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787828467.559920:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787828467.559921:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787828467.559923:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787828467.559924:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787828467.559926:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787828467.559927:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787828467.559929:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2d2c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646732928:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787828467.559932:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1787828467.559934:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1787828467.559935:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787828467.559936:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1787828467.559939:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828467.559943:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787828467.559943:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417873088 : -126234291678528 : ffff8d30c9e2d2c0) 00000100:00000001:3.0:1787828467.559947:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1787828467.559956:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:3.0:1787828467.559960:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787828467.559961:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787828467.559962:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828467.559963:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1787828467.559966:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828467.559969:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4297940503 00000100:00000001:3.0:1787828467.559970:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4297941760 00000100:00000001:3.0:1787828467.559972:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4297940503 00000100:00000001:3.0:1787828467.559974:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4297941760 00000100:00000001:3.0:1787828467.559975:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787828467.559977:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828467.559978:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1787828467.559981:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828467/real 1787828467] req@ffff8d30c9e2d2c0 x1874670646732928/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828492 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1787828467.559986:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787828467.559986:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787828467.559988:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787828467.559989:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787828467.559990:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787828467.560003:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787828467.560004:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787828467.560007:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2d2c0 x1874670646732928/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828492 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787828467.560012:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787828467.560013:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787828467.560015:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787828467.560016:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787828467.560017:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787828467.560018:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787828467.560019:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787828467.560020:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828467.560022:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787828467.560024:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787828467.560026:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2d2c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646732928:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787828467.560028:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787828467.560029:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787828467.560030:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787828467.560032:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787828467.560033:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787828467.560034:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787828467.560036:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646733056:192.168.203.143@tcp:8:kworker.0 00000100:00000001:3.0:1787828467.560038:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1787828467.560040:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1787828467.560041:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787828467.560043:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1787828467.560044:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828467.560047:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787828467.560048:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417871168 : -126234291680448 : ffff8d30c9e2cb40) 00000100:00000001:3.0:1787828467.560051:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1787828467.560080:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:3.0:1787828467.560086:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787828467.560089:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787828467.560091:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828467.560093:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1787828467.560096:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828467.560099:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4297940516 00000100:00000001:3.0:1787828467.560103:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4297941760 00000100:00000001:3.0:1787828467.560105:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4297940516 00000100:00000001:3.0:1787828467.560108:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4297941760 00000100:00000001:3.0:1787828467.560111:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787828467.560113:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828467.560115:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1787828467.560120:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828467/real 1787828467] req@ffff8d30c9e2cb40 x1874670646733056/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828492 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1787828467.560129:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787828467.560130:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787828467.560134:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787828467.560136:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787828467.560137:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787828467.560139:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787828467.560141:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787828467.560145:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2cb40 x1874670646733056/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828492 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787828467.560153:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787828467.560156:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787828467.560159:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787828467.560161:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787828467.560162:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787828467.560164:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787828467.560166:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787828467.560169:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828467.560174:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787828467.560176:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787828467.560180:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646733056:192.168.203.143@tcp:8:kworker.0 00000100:00000001:3.0:1787828467.560185:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787828467.560187:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787828467.560188:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787828467.560190:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787828467.560192:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787828467.560194:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787828467.560195:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787828467.560198:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787828467.560199:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787828467.560202:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787828467.560204:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787828467.560207:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787828467.560209:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787828467.560211:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787828467.560214:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787828467.560216:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787828467.560218:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787828467.560221:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787828467.560223:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787828467.560227:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787828467.560230:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787828467.560233:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787828467.560239:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787828467.560242:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828467.560246:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1787828467.560250:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828468.454724:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828468.454731:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787828468.454782:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:0.0:1787828468.454787:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:0.0:1787828468.454791:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828469.478751:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828469.478759:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787828469.478838:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787828469.478846:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787828469.478852:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828470.502888:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828470.502901:0:815:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1787828470.502905:0:815:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1787828470.502909:0:815:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828470.502917:0:815:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1787828470.502920:0:815:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828470.502923:0:815:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828470.502926:0:815:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787828470.503051:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787828470.503060:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787828470.503068:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828471.526704:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828471.526712:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828471.526769:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828471.526793:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828471.526808:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828472.550681:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828472.550686:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787828472.550851:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787828472.550859:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787828472.550867:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787828472.678745:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828472.678760:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1787828472.678784:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787828472.678791:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787828472.678798:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787828472.678809:0:40701:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 3273 00000100:00080000:2.0:1787828472.678825:0:40701:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787828472.678831:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828472.678835:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787828472.678842:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787828472.678846:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787828472.678851:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:2.0:1787828472.678856:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:2.0:1787828472.678860:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828472.678863:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:2.0:1787828472.678872:0:40701:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787828472.678878:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828472.678882:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787828472.678888:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828472.678893:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828472.678898:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:2.0:1787828472.678902:0:40701:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828472.678917:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787828472.678920:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828472.678926:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787828472.678930:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787828472.678936:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828472.678953:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787828472.678967:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2d680 x1874670646733184/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787828472.678988:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787828472.678995:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828472.679019:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787828472.679025:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828472.679047:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1787828472.679050:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787828472.679054:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787828472.679061:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787828472.679067:0:40701:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3273 00000100:00080000:2.0:1787828472.679074:0:40701:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787828472.679079:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828472.679082:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787828472.679087:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787828472.679089:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787828472.679094:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000020:00000001:2.0:1787828472.679099:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:3.0:1787828472.679101:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787828472.679105:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787828472.679110:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1787828472.679112:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:3.0:1787828472.679118:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00080000:2.0:1787828472.679125:0:40701:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:3.0:1787828472.679128:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787828472.679134:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828472.679137:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828472.679141:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787828472.679151:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787828472.679154:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787828472.679162:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828472.679164:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828472.679176:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 02000000:00000001:3.0:1787828472.679179:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000008:00100000:2.0:1787828472.679185:0:40701:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787828472.679194:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2d680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646733184:192.168.203.143@tcp:250:kworker.0 00000008:00000001:2.0:1787828472.679197:0:40701:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828472.679205:0:40701:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828472.679216:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787828472.679218:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828472.679222:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:3.0:1787828472.679229:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787828472.679230:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787828472.679238:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787828472.679240:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1787828472.679246:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828472.679267:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787828472.679268:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1787828472.679278:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787828472.679285:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2e1c0 x1874670646733312/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787828472.679299:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787828472.679304:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417874048 : -126234291677568 : ffff8d30c9e2d680) 00000100:00000001:3.0:1787828472.679313:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:2.0:1787828472.679323:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787828472.679328:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828472.679335:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787828472.679339:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828472.679345:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1787828472.679348:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787828472.679351:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787828472.679356:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000800:00000100:3.0:1787828472.679362:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00080000:2.0:1787828472.679367:0:40701:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3273 00000100:00080000:2.0:1787828472.679375:0:40701:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787828472.679379:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787828472.679381:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1787828472.679383:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:3.0:1787828472.679390:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000020:00000001:2.0:1787828472.679391:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787828472.679395:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787828472.679398:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000020:00000001:2.0:1787828472.679401:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:3.0:1787828472.679407:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828472.679408:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787828472.679413:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1787828472.679416:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:3.0:1787828472.679422:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787828472.679430:0:40701:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:3.0:1787828472.679432:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4297946880 00000100:00000001:2.0:1787828472.679443:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828472.679448:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:3.0:1787828472.679449:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 02000000:00000001:2.0:1787828472.679455:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787828472.679459:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4297946880 02000000:00000001:2.0:1787828472.679462:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828472.679467:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 00000100:00000001:2.0:1787828472.679469:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1787828472.679477:0:40701:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00000001:3.0:1787828472.679478:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000008:00000001:2.0:1787828472.679506:0:40701:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828472.679511:0:40701:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828472.679513:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828472.679520:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:3.0:1787828472.679524:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 02000000:00000001:2.0:1787828472.679527:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828472.679535:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787828472.679539:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00100000:3.0:1787828472.679545:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828472/real 1787828472] req@ffff8d30c9e2d680 x1874670646733184/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828497 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1787828472.679546:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828472.679558:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787828472.679577:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2cb40 x1874670646733440/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000001:3.0:1787828472.679583:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787828472.679605:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1787828472.679614:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787828472.679618:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828472.679624:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787828472.679629:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828472.679635:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1787828472.679638:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787828472.679641:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787828472.679674:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787828472.679681:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787828472.679684:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787828472.679699:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787828472.679702:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787828472.679712:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2d680 x1874670646733184/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828497 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787828472.679720:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:3.0:1787828472.679730:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:2.0:1787828472.679736:0:40701:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3273 00000100:00000001:3.0:1787828472.679756:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787828472.679795:0:40701:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787828472.679795:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787828472.679801:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828472.679804:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787828472.679805:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828472.679812:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:3.0:1787828472.679813:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787828472.679818:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000020:00000001:2.0:1787828472.679822:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000100:00080000:3.0:1787828472.679825:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000020:00000001:2.0:1787828472.679827:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787828472.679835:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00080000:3.0:1787828472.679838:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000020:00000001:2.0:1787828472.679843:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:3.0:1787828472.679845:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000001:2.0:1787828472.679871:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00100000:3.0:1787828472.679875:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2d680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646733184:192.168.203.143@tcp:250:kworker.0 00000100:00000001:2.0:1787828472.679878:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:3.0:1787828472.679893:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:2.0:1787828472.679895:0:40701:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:3.0:1787828472.679898:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828472.679907:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828472.679908:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787828472.679914:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1787828472.679917:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787828472.679925:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828472.679930:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787828472.679933:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00000001:2.0:1787828472.679935:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000020:00000001:3.0:1787828472.679942:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000008:00100000:2.0:1787828472.679943:0:40701:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000020:00000001:3.0:1787828472.679950:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 00000008:00000001:2.0:1787828472.679954:0:40701:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828472.679960:0:40701:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787828472.679962:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787828472.679969:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787828472.679970:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 02000000:00000001:2.0:1787828472.679977:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828472.679984:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:3.0:1787828472.679984:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787828472.679994:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:2.0:1787828472.679995:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787828472.680001:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828472.680005:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828472.680013:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828472.680044:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00100000:2.0:1787828472.680052:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2d2c0 x1874670646733568/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787828472.680052:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828472.680063:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1787828472.680067:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787828472.680071:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787828472.680075:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787828472.680078:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828472.680084:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 02000000:00000001:3.0:1787828472.680085:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787828472.680091:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787828472.680093:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787828472.680096:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828472.680104:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787828472.680106:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646733312:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787828472.680114:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1787828472.680119:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1787828472.680122:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787828472.680126:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1787828472.680132:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828472.680145:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787828472.680148:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417876928 : -126234291674688 : ffff8d30c9e2e1c0) 00000100:00000001:3.0:1787828472.680155:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1787828472.680182:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:3.0:1787828472.680191:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787828472.680196:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787828472.680199:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828472.680202:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1787828472.680208:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828472.680214:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4297945623 00000100:00000001:3.0:1787828472.680217:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4297946880 00000100:00000001:3.0:1787828472.680220:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4297945623 00000100:00000001:3.0:1787828472.680222:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4297946880 00000100:00000001:3.0:1787828472.680224:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787828472.680227:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828472.680229:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1787828472.680235:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828472/real 1787828472] req@ffff8d30c9e2e1c0 x1874670646733312/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828497 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1787828472.680245:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787828472.680247:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787828472.680250:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787828472.680252:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787828472.680254:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787828472.680256:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787828472.680258:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787828472.680262:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2e1c0 x1874670646733312/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828497 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787828472.680278:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787828472.680280:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787828472.680283:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787828472.680285:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787828472.680286:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787828472.680288:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787828472.680290:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787828472.680293:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828472.680297:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787828472.680299:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787828472.680302:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646733312:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787828472.680306:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787828472.680308:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787828472.680309:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787828472.680312:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787828472.680314:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787828472.680316:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787828472.680319:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646733440:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787828472.680323:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1787828472.680325:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1787828472.680327:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787828472.680328:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1787828472.680332:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828472.680337:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787828472.680339:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417871168 : -126234291680448 : ffff8d30c9e2cb40) 00000100:00000001:3.0:1787828472.680343:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1787828472.680357:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:3.0:1787828472.680362:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787828472.680364:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787828472.680366:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828472.680368:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1787828472.680371:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828472.680375:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4297945623 00000100:00000001:3.0:1787828472.680378:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4297946880 00000100:00000001:3.0:1787828472.680380:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4297945623 00000100:00000001:3.0:1787828472.680382:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4297946880 00000100:00000001:3.0:1787828472.680385:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787828472.680387:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828472.680388:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1787828472.680392:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828472/real 1787828472] req@ffff8d30c9e2cb40 x1874670646733440/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828497 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1787828472.680400:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787828472.680402:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787828472.680404:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787828472.680405:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787828472.680407:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787828472.680408:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787828472.680410:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787828472.680414:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2cb40 x1874670646733440/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828497 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787828472.680421:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787828472.680423:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787828472.680426:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787828472.680428:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787828472.680429:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787828472.680431:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787828472.680432:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787828472.680435:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828472.680438:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787828472.680440:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787828472.680444:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646733440:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787828472.680448:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787828472.680450:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787828472.680452:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787828472.680454:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787828472.680456:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787828472.680458:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787828472.680459:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787828472.680461:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787828472.680481:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787828472.680486:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787828472.680489:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787828472.680492:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787828472.680494:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787828472.680497:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787828472.680500:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787828472.680502:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787828472.680504:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787828472.680507:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787828472.680509:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787828472.680513:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787828472.680516:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787828472.680520:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787828472.680527:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787828472.680529:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828472.680534:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1787828472.680537:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787828472.680540:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787828472.680542:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787828472.680545:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787828472.680550:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787828472.680554:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787828472.680558:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787828472.680565:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2d2c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646733568:192.168.203.143@tcp:8:kworker.0 00000100:00000001:3.0:1787828472.680572:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1787828472.680576:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1787828472.680578:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787828472.680582:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1787828472.680587:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828472.680599:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787828472.680602:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417873088 : -126234291678528 : ffff8d30c9e2d2c0) 00000100:00000001:3.0:1787828472.680624:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1787828472.680648:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:3.0:1787828472.680709:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787828472.680715:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787828472.680718:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828472.680722:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1787828472.680729:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828472.680734:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4297945637 00000100:00000001:3.0:1787828472.680738:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4297946880 00000100:00000001:3.0:1787828472.680741:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4297945637 00000100:00000001:3.0:1787828472.680743:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4297946880 00000100:00000001:3.0:1787828472.680746:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787828472.680749:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828472.680751:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1787828472.680758:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828472/real 1787828472] req@ffff8d30c9e2d2c0 x1874670646733568/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828497 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1787828472.680792:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787828472.680794:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787828472.680798:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787828472.680800:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787828472.680802:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787828472.680804:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787828472.680806:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787828472.680812:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2d2c0 x1874670646733568/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828497 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787828472.680821:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787828472.680824:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787828472.680827:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787828472.680829:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787828472.680831:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787828472.680834:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787828472.680836:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787828472.680840:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828472.680843:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787828472.680846:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787828472.680850:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2d2c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646733568:192.168.203.143@tcp:8:kworker.0 00000100:00000001:3.0:1787828472.680855:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787828472.680857:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787828472.680859:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787828472.680861:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787828472.680864:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787828472.680866:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787828472.680868:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787828472.680870:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787828472.680872:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787828472.680875:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787828472.680877:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787828472.680879:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787828472.680884:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787828472.680886:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828472.680888:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1787828472.680890:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828473.574710:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828473.574718:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787828473.574813:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787828473.574818:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787828473.574823:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828474.598722:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828474.598728:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828474.598830:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828474.598834:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828474.598837:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828475.622729:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828475.622737:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787828475.622818:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787828475.622822:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787828475.622825:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828476.646665:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828476.646671:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787828476.646775:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787828476.646779:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787828476.646782:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828477.670719:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828477.670727:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828477.670807:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828477.670812:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828477.670816:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787828477.798800:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828477.798812:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1787828477.798816:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787828477.798820:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787828477.798825:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787828477.798834:0:468:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 3278 00000100:00080000:3.0:1787828477.798847:0:468:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787828477.798850:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787828477.798853:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:3.0:1787828477.798858:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1787828477.798862:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1787828477.798866:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:3.0:1787828477.798869:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:3.0:1787828477.798872:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787828477.798874:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:3.0:1787828477.798880:0:468:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:3.0:1787828477.798884:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787828477.798888:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787828477.798891:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787828477.798895:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828477.798898:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:3.0:1787828477.798901:0:468:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787828477.798913:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787828477.798915:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828477.798919:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787828477.798922:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787828477.798926:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828477.798938:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787828477.798948:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c7016d00 x1874670646733696/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787828477.798962:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787828477.798967:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828477.798987:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787828477.798995:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828477.799000:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1787828477.799002:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787828477.799004:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787828477.799008:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787828477.799012:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3278 00000100:00080000:3.0:1787828477.799018:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787828477.799021:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787828477.799024:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:3.0:1787828477.799027:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:1.0:1787828477.799030:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:3.0:1787828477.799031:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1787828477.799036:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:1.0:1787828477.799038:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000020:00000001:3.0:1787828477.799040:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:1.0:1787828477.799042:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787828477.799046:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787828477.799047:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787828477.799050:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828477.799051:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 02000000:00000001:1.0:1787828477.799058:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000100:00080000:3.0:1787828477.799059:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 02000000:00000001:1.0:1787828477.799063:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787828477.799065:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828477.799067:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787828477.799070:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00100000:1.0:1787828477.799076:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c7016d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646733696:192.168.203.143@tcp:250:kworker.0 00000100:00000001:1.0:1787828477.799082:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1787828477.799085:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787828477.799086:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787828477.799091:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787828477.799092:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828477.799096:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:3.0:1787828477.799098:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 02000000:00000001:1.0:1787828477.799103:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000008:00100000:3.0:1787828477.799105:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1787828477.799111:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828477.799115:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828477.799119:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 02000000:00000001:3.0:1787828477.799121:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:1.0:1787828477.799123:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369547008 : -126234340004608 : ffff8d30c7016d00) 02000000:00000001:3.0:1787828477.799125:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828477.799130:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:3.0:1787828477.799131:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787828477.799134:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787828477.799137:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828477.799145:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787828477.799152:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c70170c0 x1874670646733824/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787828477.799164:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787828477.799167:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:1.0:1787828477.799182:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:3.0:1787828477.799188:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787828477.799193:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828477.799200:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1787828477.799203:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787828477.799205:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787828477.799207:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787828477.799210:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:1.0:1787828477.799214:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787828477.799218:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787828477.799218:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3278 00000100:00000001:1.0:1787828477.799225:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00080000:3.0:1787828477.799228:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1787828477.799232:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828477.799232:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787828477.799238:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:1.0:1787828477.799240:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4297952000 00000020:00000001:3.0:1787828477.799243:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1787828477.799246:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1787828477.799249:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:1.0:1787828477.799253:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 00000020:00000001:3.0:1787828477.799253:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:3.0:1787828477.799260:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787828477.799262:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:3.0:1787828477.799266:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:1.0:1787828477.799268:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4297952000 00000100:00000001:3.0:1787828477.799270:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828477.799275:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 02000000:00000001:3.0:1787828477.799275:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1787828477.799282:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787828477.799282:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787828477.799288:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787828477.799289:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828477.799292:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00000001:3.0:1787828477.799295:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:3.0:1787828477.799299:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828477.799302:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828477/real 1787828477] req@ffff8d30c7016d00 x1874670646733696/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828502 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00000001:3.0:1787828477.799304:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828477.799311:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787828477.799320:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000400:00000001:1.0:1787828477.799322:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 02000000:00000001:3.0:1787828477.799324:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:1.0:1787828477.799325:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787828477.799330:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:1.0:1787828477.799331:0:2390:0:(events.c:78:reply_in_callback()) Process entered 02000000:00000001:3.0:1787828477.799333:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:1.0:1787828477.799336:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 02000000:00000001:3.0:1787828477.799338:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828477.799339:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828477.799344:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787828477.799345:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828477.799346:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787828477.799362:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c7016d00 x1874670646733696/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828502 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828477.799362:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c7017840 x1874670646733952/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787828477.799402:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787828477.799403:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787828477.799409:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828477.799410:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787828477.799416:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787828477.799416:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828477.799423:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00080000:3.0:1787828477.799424:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1787828477.799425:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828477.799430:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00080000:3.0:1787828477.799433:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1787828477.799434:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787828477.799439:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1787828477.799441:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828477.799443:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787828477.799447:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787828477.799450:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:1.0:1787828477.799451:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00080000:3.0:1787828477.799457:0:468:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3278 00000100:00100000:1.0:1787828477.799459:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c7016d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646733696:192.168.203.143@tcp:250:kworker.0 00000100:00080000:3.0:1787828477.799465:0:468:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1787828477.799472:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828477.799475:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787828477.799476:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787828477.799480:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 00000100:00000001:3.0:1787828477.799480:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 02000000:00000001:1.0:1787828477.799484:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00000001:3.0:1787828477.799484:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:1.0:1787828477.799488:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787828477.799490:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:1.0:1787828477.799492:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 00000020:00000001:3.0:1787828477.799494:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 02000000:00000001:1.0:1787828477.799497:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 00000020:00000001:3.0:1787828477.799498:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 02000000:00000001:1.0:1787828477.799500:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000020:00000001:3.0:1787828477.799505:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:1.0:1787828477.799506:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787828477.799509:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787828477.799511:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787828477.799513:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787828477.799515:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:3.0:1787828477.799522:0:468:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:1.0:1787828477.799523:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787828477.799526:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828477.799529:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828477.799530:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:3.0:1787828477.799535:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1787828477.799536:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 02000000:00000001:3.0:1787828477.799541:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787828477.799547:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 02000000:00000001:3.0:1787828477.799548:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828477.799550:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787828477.799553:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:1.0:1787828477.799554:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828477.799559:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000008:00100000:3.0:1787828477.799559:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 02000000:00000001:1.0:1787828477.799562:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000008:00000001:3.0:1787828477.799566:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828477.799567:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828477.799572:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787828477.799575:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c70170c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646733824:192.168.203.143@tcp:38:kworker.0 02000000:00000001:3.0:1787828477.799579:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:1.0:1787828477.799580:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1787828477.799582:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828477.799584:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:3.0:1787828477.799588:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1787828477.799589:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787828477.799610:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1787828477.799612:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787828477.799659:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828477.799670:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 02000000:00000001:3.0:1787828477.799676:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828477.799678:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369547968 : -126234340003648 : ffff8d30c70170c0) 00000100:00000001:3.0:1787828477.799692:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828477.799693:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00100000:3.0:1787828477.799709:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c7014000 x1874670646734080/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787828477.799727:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787828477.799766:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:1.0:1787828477.799772:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:3.0:1787828477.799774:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828477.799783:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787828477.799790:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787828477.799792:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828477.799796:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787828477.799801:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828477.799808:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4297950743 00000100:00000001:1.0:1787828477.799812:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4297952000 00000100:00000001:1.0:1787828477.799815:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4297950743 00000100:00000001:1.0:1787828477.799819:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4297952000 00000100:00000001:1.0:1787828477.799821:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787828477.799825:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828477.799827:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787828477.799834:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828477/real 1787828477] req@ffff8d30c70170c0 x1874670646733824/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828502 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787828477.799847:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787828477.799849:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787828477.799853:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787828477.799855:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787828477.799857:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828477.799859:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787828477.799861:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787828477.799867:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c70170c0 x1874670646733824/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828502 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787828477.799877:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828477.799880:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787828477.799884:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828477.799886:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787828477.799888:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828477.799890:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787828477.799893:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787828477.799896:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828477.799901:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787828477.799904:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787828477.799908:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c70170c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646733824:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787828477.799917:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787828477.799918:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787828477.799920:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828477.799922:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787828477.799924:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787828477.799926:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787828477.799928:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c7017840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646733952:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787828477.799940:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787828477.799941:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787828477.799943:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828477.799945:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787828477.799948:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828477.799953:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787828477.799955:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369549888 : -126234340001728 : ffff8d30c7017840) 00000100:00000001:1.0:1787828477.799958:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1787828477.799971:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787828477.799975:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787828477.799977:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787828477.799978:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828477.799980:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787828477.799982:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828477.799985:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4297950743 00000100:00000001:1.0:1787828477.799987:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4297952000 00000100:00000001:1.0:1787828477.799989:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4297950743 00000100:00000001:1.0:1787828477.799991:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4297952000 00000100:00000001:1.0:1787828477.799993:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787828477.799994:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828477.799996:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787828477.799999:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828477/real 1787828477] req@ffff8d30c7017840 x1874670646733952/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828502 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787828477.800006:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787828477.800008:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787828477.800009:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787828477.800011:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787828477.800012:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828477.800013:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787828477.800015:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787828477.800018:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c7017840 x1874670646733952/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828502 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787828477.800023:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828477.800025:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787828477.800027:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828477.800029:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787828477.800030:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828477.800031:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787828477.800032:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787828477.800034:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828477.800037:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787828477.800038:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787828477.800041:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c7017840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646733952:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787828477.800044:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828477.800045:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787828477.800046:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787828477.800047:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787828477.800049:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787828477.800050:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787828477.800051:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787828477.800053:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787828477.800054:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787828477.800055:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787828477.800056:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828477.800058:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787828477.800059:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787828477.800060:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787828477.800061:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787828477.800063:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787828477.800064:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787828477.800066:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787828477.800067:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787828477.800068:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787828477.800069:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828477.800071:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828477.800074:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787828477.800075:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828477.800076:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787828477.800078:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1787828477.800079:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787828477.800080:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787828477.800081:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828477.800083:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787828477.800085:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787828477.800086:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787828477.800089:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c7014000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646734080:192.168.203.143@tcp:8:kworker.0 00000100:00000001:1.0:1787828477.800091:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787828477.800092:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787828477.800094:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828477.800095:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787828477.800097:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828477.800101:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787828477.800103:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369535488 : -126234340016128 : ffff8d30c7014000) 00000100:00000001:1.0:1787828477.800106:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1787828477.800114:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787828477.800117:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787828477.800118:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787828477.800119:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828477.800121:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787828477.800123:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828477.800125:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4297950756 00000100:00000001:1.0:1787828477.800127:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4297952000 00000100:00000001:1.0:1787828477.800129:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4297950756 00000100:00000001:1.0:1787828477.800131:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4297952000 00000100:00000001:1.0:1787828477.800134:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787828477.800135:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828477.800137:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787828477.800140:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828477/real 1787828477] req@ffff8d30c7014000 x1874670646734080/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828502 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787828477.800146:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787828477.800147:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787828477.800149:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787828477.800150:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787828477.800151:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828477.800153:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787828477.800154:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787828477.800157:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c7014000 x1874670646734080/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828502 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787828477.800162:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828477.800163:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787828477.800165:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828477.800166:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787828477.800167:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828477.800168:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787828477.800170:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787828477.800172:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828477.800179:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787828477.800181:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787828477.800183:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c7014000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646734080:192.168.203.143@tcp:8:kworker.0 00000100:00000001:1.0:1787828477.800186:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828477.800187:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787828477.800188:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787828477.800189:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787828477.800191:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787828477.800192:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787828477.800193:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787828477.800195:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787828477.800196:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787828477.800197:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787828477.800198:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828477.800200:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828477.800202:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787828477.800203:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828477.800204:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787828477.800206:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828478.694676:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828478.694682:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787828478.694753:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787828478.694758:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787828478.694762:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828479.718705:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828479.718711:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787828479.718847:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787828479.718854:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787828479.718860:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828480.742706:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828480.742738:0:815:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1787828480.742741:0:815:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1787828480.742744:0:815:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828480.742749:0:815:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1787828480.742752:0:815:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828480.742754:0:815:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828480.742756:0:815:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828480.742827:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828480.742832:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828480.742837:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828481.766831:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828481.766841:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787828481.766945:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787828481.766954:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787828481.766961:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828482.790673:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828482.790678:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787828482.790753:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787828482.790757:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787828482.790761:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787828482.918774:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828482.918783:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1787828482.918786:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787828482.918790:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787828482.918795:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787828482.918803:0:40701:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 3283 00000100:00080000:0.0:1787828482.918813:0:40701:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787828482.918816:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787828482.918818:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787828482.918822:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787828482.918824:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787828482.918827:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:0.0:1787828482.918829:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:0.0:1787828482.918831:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787828482.918832:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:0.0:1787828482.918836:0:40701:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787828482.918838:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828482.918840:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787828482.918843:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787828482.918845:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828482.918847:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1787828482.918850:0:40701:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828482.918863:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787828482.918865:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828482.918869:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787828482.918888:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787828482.918892:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828482.918905:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828482.918914:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d252c0 x1874670646734208/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787828482.918924:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787828482.918927:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828482.918938:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787828482.918941:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828482.918943:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1787828482.918944:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787828482.918946:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787828482.918950:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787828482.918952:0:40701:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3283 00000100:00080000:0.0:1787828482.918957:0:40701:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787828482.918960:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787828482.918961:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787828482.918964:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787828482.918965:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787828482.918968:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000020:00000001:0.0:1787828482.918972:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:0.0:1787828482.918974:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787828482.918976:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:0.0:1787828482.918981:0:40701:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787828482.918984:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828482.918986:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1787828482.918986:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787828482.918993:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 02000000:00000001:0.0:1787828482.918996:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787828482.918997:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 02000000:00000001:0.0:1787828482.918999:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828482.919002:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787828482.919003:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:1.0:1787828482.919008:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000008:00100000:0.0:1787828482.919009:0:40701:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1787828482.919014:0:40701:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828482.919016:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:0.0:1787828482.919017:0:40701:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828482.919022:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1787828482.919023:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787828482.919024:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828482.919028:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787828482.919029:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1787828482.919029:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787828482.919038:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d252c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646734208:192.168.203.143@tcp:250:kworker.0 02000000:00000001:0.0:1787828482.919039:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828482.919047:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:0.0:1787828482.919050:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828482.919053:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787828482.919058:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828482.919060:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d26940 x1874670646734336/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:1.0:1787828482.919062:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787828482.919069:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828482.919073:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787828482.919078:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828482.919083:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787828482.919086:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1787828482.919088:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787828482.919091:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601341120 : -126234108210496 : ffff8d30d4d252c0) 00000100:00080000:0.0:1787828482.919093:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1787828482.919098:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787828482.919100:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787828482.919101:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:0.0:1787828482.919103:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787828482.919107:0:40701:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3283 00000100:00080000:0.0:1787828482.919111:0:40701:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787828482.919112:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787828482.919113:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787828482.919115:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787828482.919117:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787828482.919119:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000020:00000001:0.0:1787828482.919120:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:0.0:1787828482.919133:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787828482.919134:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:0.0:1787828482.919138:0:40701:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787828482.919141:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828482.919143:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787828482.919147:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787828482.919151:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:1.0:1787828482.919152:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:0.0:1787828482.919155:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1787828482.919159:0:40701:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1787828482.919163:0:40701:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828482.919166:0:40701:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828482.919167:0:2390:0:(events.c:34:request_out_callback()) Process entered 02000000:00000001:0.0:1787828482.919172:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:1.0:1787828482.919174:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 02000000:00000001:0.0:1787828482.919176:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828482.919178:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828482.919180:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787828482.919182:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787828482.919185:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828482.919185:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787828482.919193:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828482.919196:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828482.919199:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d27c00 x1874670646734464/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787828482.919217:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4297957120 00000100:00000001:1.0:1787828482.919232:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1787828482.919233:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787828482.919238:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828482.919242:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828482.919244:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4297957120 00000100:00080000:0.0:1787828482.919245:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828482.919252:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1787828482.919254:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:1.0:1787828482.919256:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 00000100:00080000:0.0:1787828482.919257:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787828482.919265:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:1.0:1787828482.919267:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00080000:0.0:1787828482.919268:0:40701:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3283 00000100:00080000:0.0:1787828482.919281:0:40701:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787828482.919286:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787828482.919286:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828482.919289:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787828482.919293:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:1.0:1787828482.919293:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000020:00000001:0.0:1787828482.919294:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787828482.919299:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00100000:1.0:1787828482.919306:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828482/real 1787828482] req@ffff8d30d4d252c0 x1874670646734208/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828507 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:0.0:1787828482.919311:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:0.0:1787828482.919316:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787828482.919318:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:0.0:1787828482.919329:0:40701:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000400:00000001:1.0:1787828482.919335:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000001:0.0:1787828482.919336:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:1.0:1787828482.919356:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 02000000:00000001:0.0:1787828482.919357:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787828482.919363:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787828482.919363:0:2390:0:(events.c:78:reply_in_callback()) Process entered 02000000:00000001:0.0:1787828482.919368:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828482.919368:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787828482.919373:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828482.919374:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1787828482.919379:0:40701:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00000001:1.0:1787828482.919379:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787828482.919383:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000008:00000001:0.0:1787828482.919386:0:40701:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828482.919389:0:40701:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787828482.919393:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d252c0 x1874670646734208/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828507 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:0.0:1787828482.919394:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787828482.919397:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828482.919400:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787828482.919402:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787828482.919405:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828482.919412:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828482.919413:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828482.919416:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00100000:0.0:1787828482.919423:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d25680 x1874670646734592/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787828482.919424:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828482.919429:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787828482.919431:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828482.919437:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787828482.919441:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787828482.919442:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787828482.919448:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1787828482.919449:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00000001:0.0:1787828482.919452:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1787828482.919458:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787828482.919462:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787828482.919467:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d252c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646734208:192.168.203.143@tcp:250:kworker.0 00000100:00000001:1.0:1787828482.919483:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828482.919492:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787828482.919494:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787828482.919496:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787828482.919500:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787828482.919502:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787828482.919504:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787828482.919508:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787828482.919509:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787828482.919512:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787828482.919514:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828482.919516:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828482.919522:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787828482.919524:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828482.919527:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787828482.919528:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1787828482.919530:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787828482.919531:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787828482.919532:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828482.919535:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787828482.919536:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787828482.919538:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787828482.919542:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d26940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646734336:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787828482.919545:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787828482.919547:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787828482.919564:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828482.919576:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787828482.919579:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828482.919586:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787828482.919588:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601346880 : -126234108204736 : ffff8d30d4d26940) 00000100:00000001:1.0:1787828482.919593:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1787828482.919609:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787828482.919646:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787828482.919650:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787828482.919652:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828482.919655:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787828482.919659:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828482.919664:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4297955863 00000100:00000001:1.0:1787828482.919668:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4297957120 00000100:00000001:1.0:1787828482.919670:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4297955863 00000100:00000001:1.0:1787828482.919673:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4297957120 00000100:00000001:1.0:1787828482.919676:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787828482.919679:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828482.919681:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787828482.919687:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828482/real 1787828482] req@ffff8d30d4d26940 x1874670646734336/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828507 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787828482.919713:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787828482.919715:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787828482.919719:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787828482.919731:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787828482.919732:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828482.919734:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787828482.919736:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787828482.919757:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d26940 x1874670646734336/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828507 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787828482.919766:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828482.919768:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787828482.919771:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828482.919773:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787828482.919775:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828482.919777:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787828482.919779:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787828482.919782:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828482.919797:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787828482.919801:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787828482.919805:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d26940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646734336:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787828482.919809:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787828482.919811:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787828482.919813:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828482.919816:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787828482.919818:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787828482.919820:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787828482.919824:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d27c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646734464:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787828482.919827:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787828482.919830:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787828482.919831:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828482.919834:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787828482.919837:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828482.919844:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787828482.919846:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601351680 : -126234108199936 : ffff8d30d4d27c00) 00000100:00000001:1.0:1787828482.919850:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1787828482.919865:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787828482.919870:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787828482.919873:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787828482.919890:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828482.919892:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787828482.919895:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828482.919899:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4297955863 00000100:00000001:1.0:1787828482.919902:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4297957120 00000100:00000001:1.0:1787828482.919904:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4297955863 00000100:00000001:1.0:1787828482.919907:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4297957120 00000100:00000001:1.0:1787828482.919909:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787828482.919911:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828482.919913:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787828482.919917:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828482/real 1787828482] req@ffff8d30d4d27c00 x1874670646734464/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828507 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787828482.919934:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787828482.919935:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787828482.919938:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787828482.919939:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787828482.919941:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828482.919943:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787828482.919944:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787828482.919949:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d27c00 x1874670646734464/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828507 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787828482.919956:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828482.919959:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787828482.919961:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828482.919963:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787828482.919964:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828482.919967:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787828482.919968:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787828482.919971:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828482.919975:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787828482.919977:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787828482.919980:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d27c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646734464:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787828482.919984:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787828482.919986:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787828482.919987:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828482.919990:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787828482.919991:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787828482.919993:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787828482.919997:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d25680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646734592:192.168.203.143@tcp:8:kworker.0 00000100:00000001:1.0:1787828482.920000:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787828482.920002:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787828482.920004:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828482.920006:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787828482.920009:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828482.920014:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787828482.920016:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601342080 : -126234108209536 : ffff8d30d4d25680) 00000100:00000001:1.0:1787828482.920019:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1787828482.920032:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787828482.920036:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787828482.920038:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787828482.920040:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828482.920042:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787828482.920045:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828482.920048:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4297955876 00000100:00000001:1.0:1787828482.920051:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4297957120 00000100:00000001:1.0:1787828482.920053:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4297955876 00000100:00000001:1.0:1787828482.920056:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4297957120 00000100:00000001:1.0:1787828482.920058:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787828482.920060:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828482.920062:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787828482.920066:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828482/real 1787828482] req@ffff8d30d4d25680 x1874670646734592/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828507 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787828482.920074:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787828482.920076:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787828482.920078:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787828482.920091:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787828482.920093:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828482.920094:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787828482.920096:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787828482.920100:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d25680 x1874670646734592/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828507 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787828482.920107:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828482.920109:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787828482.920112:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828482.920114:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787828482.920115:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828482.920117:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787828482.920119:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787828482.920122:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828482.920125:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787828482.920127:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787828482.920130:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d25680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646734592:192.168.203.143@tcp:8:kworker.0 00000100:00000001:1.0:1787828482.920134:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828482.920136:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787828482.920137:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787828482.920139:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787828482.920141:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787828482.920142:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787828482.920144:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787828482.920146:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787828482.920147:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787828482.920150:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787828482.920152:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828482.920153:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787828482.920155:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787828482.920156:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787828482.920168:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787828482.920170:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787828482.920171:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787828482.920173:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787828482.920174:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787828482.920176:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787828482.920177:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828482.920179:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787828482.920180:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787828482.920181:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787828482.920198:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787828482.920199:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787828482.920201:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787828482.920203:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787828482.920205:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787828482.920207:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787828482.920208:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828482.920210:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828482.920214:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787828482.920216:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828482.920218:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787828482.920221:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828483.814761:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828483.814768:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828483.814833:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828483.814836:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828483.814839:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828484.838741:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828484.838746:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787828484.838807:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787828484.838811:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787828484.838814:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828485.862694:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828485.862699:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787828485.862752:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787828485.862761:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787828485.862765:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828486.886722:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828486.886728:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828486.886797:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828486.886801:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828486.886810:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828487.910714:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828487.910719:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787828487.910776:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787828487.910780:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787828487.910783:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787828488.038818:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828488.038830:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1787828488.038833:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787828488.038837:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787828488.038843:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787828488.038851:0:40701:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 3288 00000100:00080000:2.0:1787828488.038863:0:40701:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787828488.038867:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828488.038869:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787828488.038874:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787828488.038877:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787828488.038881:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:2.0:1787828488.038884:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:2.0:1787828488.038887:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828488.038889:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:2.0:1787828488.038894:0:40701:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787828488.038897:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828488.038900:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787828488.038905:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828488.038908:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828488.038912:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:2.0:1787828488.038915:0:40701:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828488.038929:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787828488.038931:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828488.038934:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787828488.038938:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787828488.038943:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828488.038956:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787828488.038968:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2fc00 x1874670646734720/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787828488.038981:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787828488.038985:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828488.039009:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787828488.039013:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828488.039025:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1787828488.039027:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787828488.039029:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787828488.039032:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787828488.039035:0:40701:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3288 00000100:00080000:2.0:1787828488.039039:0:40701:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787828488.039041:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828488.039043:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787828488.039046:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787828488.039048:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787828488.039051:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000020:00000001:2.0:1787828488.039054:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:2.0:1787828488.039057:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828488.039060:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:2.0:1787828488.039064:0:40701:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787828488.039067:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828488.039069:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787828488.039077:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787828488.039082:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:2.0:1787828488.039082:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828488.039089:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:1.0:1787828488.039092:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000008:00100000:2.0:1787828488.039095:0:40701:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1787828488.039101:0:40701:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828488.039103:0:40701:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828488.039108:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787828488.039109:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828488.039111:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1787828488.039112:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787828488.039119:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:1.0:1787828488.039120:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 02000000:00000001:2.0:1787828488.039124:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828488.039126:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828488.039133:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828488.039135:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787828488.039142:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00100000:2.0:1787828488.039145:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2de00 x1874670646734848/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:1.0:1787828488.039148:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828488.039163:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787828488.039168:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787828488.039171:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646734720:192.168.203.143@tcp:250:kworker.0 00000100:00000001:2.0:1787828488.039175:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787828488.039185:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1787828488.039186:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00080000:2.0:1787828488.039194:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 02000000:00000001:1.0:1787828488.039197:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:2.0:1787828488.039198:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787828488.039204:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 02000000:00000001:1.0:1787828488.039205:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828488.039210:0:40701:0:(import.c:485:import_select_connection()) Process entered 02000000:00000001:1.0:1787828488.039212:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00080000:2.0:1787828488.039218:0:40701:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3288 02000000:00000001:1.0:1787828488.039222:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787828488.039224:0:40701:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787828488.039231:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828488.039233:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787828488.039236:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787828488.039237:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787828488.039240:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000020:00000001:2.0:1787828488.039243:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:2.0:1787828488.039245:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787828488.039248:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787828488.039249:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:1.0:1787828488.039255:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417883648 : -126234291667968 : ffff8d30c9e2fc00) 00000100:00080000:2.0:1787828488.039260:0:40701:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:1.0:1787828488.039264:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:2.0:1787828488.039266:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828488.039288:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787828488.039292:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828488.039296:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828488.039298:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1787828488.039302:0:40701:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1787828488.039306:0:40701:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828488.039308:0:40701:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828488.039314:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787828488.039317:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828488.039319:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787828488.039321:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787828488.039323:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828488.039331:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:1.0:1787828488.039337:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00100000:2.0:1787828488.039346:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2c780 x1874670646734976/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787828488.039356:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1787828488.039359:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787828488.039365:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828488.039367:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828488.039370:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828488.039375:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828488.039379:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787828488.039388:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787828488.039389:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828488.039399:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1787828488.039401:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787828488.039404:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787828488.039408:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:1.0:1787828488.039411:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4297962240 00000100:00080000:2.0:1787828488.039412:0:40701:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3288 00000100:00080000:2.0:1787828488.039442:0:40701:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787828488.039445:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787828488.039450:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 00000100:00000001:2.0:1787828488.039450:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787828488.039457:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787828488.039461:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787828488.039464:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:1.0:1787828488.039468:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4297962240 00000020:00000001:2.0:1787828488.039469:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:2.0:1787828488.039474:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828488.039478:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:1.0:1787828488.039484:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 00000100:00080000:2.0:1787828488.039485:0:40701:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787828488.039496:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828488.039499:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1787828488.039500:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828488.039503:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828488.039511:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828488.039512:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828488.039515:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1787828488.039522:0:40701:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00000001:1.0:1787828488.039524:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000008:00000001:2.0:1787828488.039528:0:40701:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828488.039533:0:40701:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828488.039540:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00100000:1.0:1787828488.039544:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828488/real 1787828488] req@ffff8d30c9e2fc00 x1874670646734720/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828513 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1787828488.039545:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828488.039552:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787828488.039555:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787828488.039559:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828488.039569:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:1.0:1787828488.039577:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00100000:2.0:1787828488.039581:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2e940 x1874670646735104/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000001:1.0:1787828488.039583:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787828488.039591:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787828488.039597:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787828488.039598:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787828488.039603:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828488.039605:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828488.039611:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828488.039613:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828488.039690:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787828488.039702:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2fc00 x1874670646734720/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828513 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787828488.039715:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828488.039720:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787828488.039724:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828488.039726:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787828488.039728:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828488.039730:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787828488.039733:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787828488.039736:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828488.039741:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787828488.039744:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787828488.039748:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646734720:192.168.203.143@tcp:250:kworker.0 00000100:00000001:1.0:1787828488.039755:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828488.039758:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787828488.039760:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787828488.039763:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787828488.039768:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787828488.039770:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787828488.039773:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787828488.039778:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787828488.039780:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787828488.039784:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787828488.039786:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828488.039789:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828488.039797:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787828488.039799:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828488.039802:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787828488.039804:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1787828488.039806:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787828488.039808:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787828488.039810:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828488.039812:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787828488.039815:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787828488.039817:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787828488.039821:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646734848:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787828488.039825:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787828488.039828:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787828488.039830:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828488.039832:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787828488.039836:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828488.039844:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787828488.039846:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417875968 : -126234291675648 : ffff8d30c9e2de00) 00000100:00000001:1.0:1787828488.039851:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1787828488.039870:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787828488.039875:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787828488.039878:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787828488.039880:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828488.039882:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787828488.039886:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828488.039892:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4297960983 00000100:00000001:1.0:1787828488.039896:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4297962240 00000100:00000001:1.0:1787828488.039898:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4297960983 00000100:00000001:1.0:1787828488.039901:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4297962240 00000100:00000001:1.0:1787828488.039904:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787828488.039906:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828488.039909:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787828488.039944:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828488/real 1787828488] req@ffff8d30c9e2de00 x1874670646734848/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828513 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787828488.039956:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787828488.039957:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787828488.039961:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787828488.039963:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787828488.039965:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828488.039967:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787828488.039969:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787828488.039973:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2de00 x1874670646734848/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828513 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787828488.039982:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828488.039985:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787828488.039988:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828488.039990:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787828488.039991:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828488.039993:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787828488.039995:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787828488.039999:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828488.040003:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787828488.040006:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787828488.040010:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646734848:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787828488.040018:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787828488.040020:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787828488.040022:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828488.040025:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787828488.040027:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787828488.040029:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787828488.040033:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646734976:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787828488.040037:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787828488.040039:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787828488.040042:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828488.040044:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787828488.040048:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828488.040054:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787828488.040056:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417870208 : -126234291681408 : ffff8d30c9e2c780) 00000100:00000001:1.0:1787828488.040061:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1787828488.040076:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787828488.040081:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787828488.040083:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787828488.040085:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828488.040087:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787828488.040091:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828488.040095:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4297960983 00000100:00000001:1.0:1787828488.040098:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4297962240 00000100:00000001:1.0:1787828488.040101:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4297960983 00000100:00000001:1.0:1787828488.040103:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4297962240 00000100:00000001:1.0:1787828488.040106:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787828488.040108:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828488.040111:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787828488.040115:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828488/real 1787828488] req@ffff8d30c9e2c780 x1874670646734976/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828513 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787828488.040125:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787828488.040127:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787828488.040129:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787828488.040131:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787828488.040133:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828488.040135:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787828488.040137:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787828488.040142:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2c780 x1874670646734976/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828513 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787828488.040150:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828488.040152:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787828488.040155:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828488.040157:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787828488.040159:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828488.040161:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787828488.040162:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787828488.040165:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828488.040169:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787828488.040171:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787828488.040175:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646734976:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787828488.040179:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787828488.040181:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787828488.040182:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828488.040185:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787828488.040187:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787828488.040189:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787828488.040193:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646735104:192.168.203.143@tcp:8:kworker.0 00000100:00000001:1.0:1787828488.040196:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787828488.040199:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787828488.040201:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828488.040203:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787828488.040206:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828488.040211:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787828488.040213:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417878848 : -126234291672768 : ffff8d30c9e2e940) 00000100:00000001:1.0:1787828488.040217:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1787828488.040228:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787828488.040233:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787828488.040235:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787828488.040237:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828488.040239:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787828488.040242:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828488.040245:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4297960996 00000100:00000001:1.0:1787828488.040249:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4297962240 00000100:00000001:1.0:1787828488.040251:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4297960996 00000100:00000001:1.0:1787828488.040254:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4297962240 00000100:00000001:1.0:1787828488.040257:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787828488.040259:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828488.040261:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787828488.040265:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828488/real 1787828488] req@ffff8d30c9e2e940 x1874670646735104/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828513 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787828488.040274:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787828488.040276:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787828488.040278:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787828488.040280:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787828488.040282:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828488.040283:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787828488.040285:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787828488.040289:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2e940 x1874670646735104/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828513 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787828488.040298:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828488.040300:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787828488.040303:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828488.040305:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787828488.040307:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828488.040309:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787828488.040312:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787828488.040315:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828488.040319:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787828488.040322:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787828488.040325:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646735104:192.168.203.143@tcp:8:kworker.0 00000100:00000001:1.0:1787828488.040330:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828488.040332:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787828488.040333:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787828488.040335:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787828488.040338:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787828488.040339:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787828488.040341:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787828488.040344:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787828488.040345:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787828488.040351:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787828488.040353:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828488.040355:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787828488.040356:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787828488.040357:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787828488.040360:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787828488.040362:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787828488.040363:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787828488.040365:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787828488.040367:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787828488.040370:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787828488.040372:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828488.040374:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787828488.040375:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787828488.040377:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787828488.040379:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787828488.040380:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787828488.040381:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787828488.040383:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787828488.040384:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787828488.040386:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787828488.040388:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828488.040390:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828488.040393:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787828488.040396:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828488.040398:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787828488.040401:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828488.934725:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828488.934733:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787828488.934800:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787828488.934804:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787828488.934813:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828489.958747:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828489.958755:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828489.958829:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828489.958834:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828489.958839:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828490.982706:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828490.982714:0:815:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1787828490.982716:0:815:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1787828490.982719:0:815:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828490.982724:0:815:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1787828490.982726:0:815:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828490.982741:0:815:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828490.982743:0:815:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787828490.982812:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787828490.982817:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787828490.982820:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828492.006709:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828492.006715:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787828492.006787:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787828492.006792:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787828492.006796:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828493.030692:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828493.030698:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828493.030745:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828493.030749:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828493.030757:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787828493.158728:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828493.158735:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1787828493.158737:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787828493.158740:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787828493.158743:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787828493.158749:0:468:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 3293 00000100:00080000:0.0:1787828493.158757:0:468:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787828493.158759:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787828493.158761:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787828493.158764:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787828493.158766:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787828493.158768:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:0.0:1787828493.158770:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:0.0:1787828493.158772:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787828493.158773:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:0.0:1787828493.158776:0:468:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787828493.158778:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828493.158779:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787828493.158782:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787828493.158784:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828493.158785:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1787828493.158787:0:468:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828493.158796:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787828493.158798:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828493.158800:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787828493.158802:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787828493.158805:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828493.158815:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828493.158823:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d25680 x1874670646735232/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787828493.158830:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787828493.158834:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828493.158845:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787828493.158847:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828493.158850:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1787828493.158851:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787828493.158852:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787828493.158854:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787828493.158855:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3293 00000100:00080000:0.0:1787828493.158857:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787828493.158859:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787828493.158860:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787828493.158862:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787828493.158863:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787828493.158865:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000020:00000001:0.0:1787828493.158867:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:0.0:1787828493.158868:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787828493.158869:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:0.0:1787828493.158871:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787828493.158873:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828493.158874:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1787828493.158878:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:0.0:1787828493.158879:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787828493.158882:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828493.158884:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787828493.158885:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:1.0:1787828493.158889:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000008:00100000:0.0:1787828493.158890:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00000001:1.0:1787828493.158892:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000008:00000001:0.0:1787828493.158895:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828493.158896:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828493.158899:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828493.158902:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787828493.158903:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787828493.158906:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828493.158906:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787828493.158909:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1787828493.158911:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828493.158912:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787828493.158915:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787828493.158918:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d25680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646735232:192.168.203.143@tcp:250:kworker.0 00000100:00000001:0.0:1787828493.158920:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828493.158927:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00100000:0.0:1787828493.158929:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d27c00 x1874670646735360/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:1.0:1787828493.158932:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787828493.158935:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828493.158940:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 02000000:00000001:1.0:1787828493.158940:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:0.0:1787828493.158943:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828493.158946:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828493.158946:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787828493.158952:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828493.158955:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1787828493.158956:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787828493.158958:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787828493.158959:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787828493.158961:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3293 00000100:00000001:1.0:1787828493.158962:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00080000:0.0:1787828493.158964:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1787828493.158965:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601342080 : -126234108209536 : ffff8d30d4d25680) 00000100:00000001:0.0:1787828493.158967:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787828493.158970:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:1.0:1787828493.158972:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000020:00000001:0.0:1787828493.158973:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787828493.158976:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787828493.158977:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000020:00000001:0.0:1787828493.158979:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:0.0:1787828493.158980:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787828493.158982:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:0.0:1787828493.158984:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787828493.158986:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828493.158988:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787828493.158990:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787828493.158991:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828493.158992:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1787828493.158993:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1787828493.158995:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828493.158997:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828493.159000:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787828493.159001:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828493.159002:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787828493.159003:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787828493.159004:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828493.159007:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828493.159010:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d26940 x1874670646735488/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787828493.159013:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828493.159018:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787828493.159021:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828493.159021:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4297967360 00000100:00000001:0.0:1787828493.159024:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787828493.159028:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828493.159031:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1787828493.159032:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787828493.159033:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787828493.159035:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:1.0:1787828493.159036:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 00000100:00080000:0.0:1787828493.159038:0:468:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3293 00000100:00080000:0.0:1787828493.159043:0:468:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787828493.159045:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787828493.159046:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787828493.159048:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:1.0:1787828493.159048:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4297967360 00000020:00000001:0.0:1787828493.159051:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787828493.159053:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000020:00000001:0.0:1787828493.159056:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:1.0:1787828493.159057:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1787828493.159058:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787828493.159060:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:0.0:1787828493.159065:0:468:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:1.0:1787828493.159065:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787828493.159067:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828493.159072:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1787828493.159074:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828493.159075:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787828493.159079:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828493.159079:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828493.159082:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:1.0:1787828493.159084:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000008:00100000:0.0:1787828493.159085:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1787828493.159089:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828493.159090:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828493.159093:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 02000000:00000001:0.0:1787828493.159094:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787828493.159098:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828493.159098:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:0.0:1787828493.159104:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787828493.159105:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:1.0:1787828493.159105:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787828493.159108:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 02000000:00000001:0.0:1787828493.159109:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828493.159112:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1787828493.159115:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828493.159115:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787828493.159118:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828493.159122:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d252c0 x1874670646735616/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:1.0:1787828493.159122:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787828493.159125:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787828493.159130:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828493.159136:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00100000:1.0:1787828493.159138:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d27c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646735360:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787828493.159139:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828493.159145:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828493.159145:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787828493.159149:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787828493.159151:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828493.159153:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787828493.159156:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828493.159162:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787828493.159164:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601351680 : -126234108199936 : ffff8d30d4d27c00) 00000100:00000001:1.0:1787828493.159167:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:1.0:1787828493.159178:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828493.159181:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4297966102 00000100:00000001:1.0:1787828493.159184:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4297967360 00000100:00000001:1.0:1787828493.159186:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4297966102 00000100:00000001:1.0:1787828493.159188:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4297967360 00000100:00000001:1.0:1787828493.159190:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787828493.159191:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828493.159193:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787828493.159194:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787828493.159195:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828493.159196:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787828493.159197:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787828493.159205:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787828493.159209:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d26940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646735488:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787828493.159211:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787828493.159213:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787828493.159214:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828493.159215:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787828493.159217:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828493.159220:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787828493.159221:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601346880 : -126234108204736 : ffff8d30d4d26940) 00000100:00000001:1.0:1787828493.159223:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:1.0:1787828493.159229:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828493.159232:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4297966102 00000100:00000001:1.0:1787828493.159233:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4297967360 00000100:00000001:1.0:1787828493.159235:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4297966102 00000100:00000001:1.0:1787828493.159236:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4297967360 00000100:00000001:1.0:1787828493.159237:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787828493.159238:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828493.159240:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828493.159241:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828493.159243:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787828493.159244:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:1.0:1787828493.159245:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787828493.159246:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1787828493.159247:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787828493.159248:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787828493.159249:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828493.159250:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787828493.159251:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787828493.159252:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787828493.159254:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d252c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646735616:192.168.203.143@tcp:8:kworker.0 00000100:00000001:1.0:1787828493.159256:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787828493.159257:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787828493.159258:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828493.159259:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787828493.159261:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828493.159264:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787828493.159265:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601341120 : -126234108210496 : ffff8d30d4d252c0) 00000100:00000001:1.0:1787828493.159267:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:1.0:1787828493.159272:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828493.159274:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4297966115 00000100:00000001:1.0:1787828493.159275:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4297967360 00000100:00000001:1.0:1787828493.159277:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4297966115 00000100:00000001:1.0:1787828493.159278:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4297967360 00000100:00000001:1.0:1787828493.159279:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787828493.159280:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828493.159282:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828493.159283:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828493.159284:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787828493.159285:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:1.0:1787828493.159286:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787828493.159287:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1787828493.159288:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828493.159289:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828494.054717:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828494.054722:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787828494.054785:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787828494.054789:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787828494.054793:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828495.078717:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828495.078724:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787828495.078795:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787828495.078801:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787828495.078807:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828496.102734:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828496.102742:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828496.102804:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828496.102808:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828496.102811:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:1.0:1787828496.231054:0:2378:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.143:988 00000400:00000100:1.0:1787828496.231142:0:2378:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.143@tcp at host 192.168.203.143 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1787828496.231167:0:2378:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.143@tcp 00000100:00000001:1.0:1787828496.231187:0:2378:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787828496.231218:0:2378:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787828496.231223:0:2378:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828496.231228:0:2378:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:1.0:1787828496.231244:0:2378:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.143@tcp 00000100:00000001:1.0:1787828496.231255:0:2378:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1787828496.231263:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787828496.231264:0:2378:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828496.231272:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1787828496.231280:0:2378:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828496.231281:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00000001:1.0:1787828496.231288:0:2378:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:1.0:1787828496.231301:0:2378:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.143@tcp 00000100:00100000:2.0:1787828496.231303:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828493/real 1787828496] req@ffff8d30d4d27c00 x1874670646735360/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828518 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787828496.231310:0:2378:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787828496.231316:0:2378:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787828496.231320:0:2378:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828496.231325:0:2378:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:1.0:1787828496.231335:0:2378:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.143@tcp 00000400:00000001:2.0:1787828496.231337:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787828496.231356:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1787828496.231366:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787828496.231370:0:2378:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1787828496.231383:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787828496.231386:0:2378:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828496.231390:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828496.231393:0:2378:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828496.231398:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787828496.231400:0:2378:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1787828496.231406:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787828496.231418:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d27c00 x1874670646735360/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828518 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787828496.231432:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828496.231437:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787828496.231442:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828496.231444:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828496.231446:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828496.231451:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787828496.231454:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787828496.231460:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828496.231468:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:2.0:1787828496.231472:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1787828496.231479:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d27c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646735360:192.168.203.143@tcp:38:kworker.0 00000100:00000001:2.0:1787828496.231486:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1787828496.231492:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828493/real 1787828496] req@ffff8d30d4d26940 x1874670646735488/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828518 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1787828496.231516:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787828496.231517:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1787828496.231523:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1787828496.231525:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787828496.231528:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828496.231530:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828496.231532:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787828496.231539:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d26940 x1874670646735488/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828518 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787828496.231550:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828496.231553:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787828496.231556:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828496.231559:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828496.231561:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828496.231564:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787828496.231567:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787828496.231570:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828496.231576:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:2.0:1787828496.231580:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1787828496.231585:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d26940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646735488:192.168.203.143@tcp:38:kworker.0 00000100:00000001:2.0:1787828496.231592:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1787828496.231697:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828493/real 1787828496] req@ffff8d30d4d25680 x1874670646735232/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828518 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1787828496.231716:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787828496.231718:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1787828496.231734:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1787828496.231738:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787828496.231741:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828496.231745:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828496.231747:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787828496.231757:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d25680 x1874670646735232/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828518 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787828496.231768:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828496.231772:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787828496.231776:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828496.231779:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828496.231782:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828496.231785:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787828496.231789:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787828496.231793:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828496.231799:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:2.0:1787828496.231803:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1787828496.231808:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d25680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646735232:192.168.203.143@tcp:250:kworker.0 00000100:00000001:2.0:1787828496.231814:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828496.231818:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828496.231820:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787828496.231824:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1787828496.231830:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787828496.231834:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:2.0:1787828496.231838:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:2.0:1787828496.231843:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787828496.231846:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1787828496.231852:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787828496.231854:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828496.231857:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828496.231859:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787828496.231861:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1787828496.231865:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787828496.231868:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:2.0:1787828496.231869:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:2.0:1787828496.231871:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787828496.231873:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1787828496.231877:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787828496.231878:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828496.231881:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828496.231883:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787828496.231885:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1787828496.231887:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787828496.231889:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:2.0:1787828496.231891:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:2.0:1787828496.231893:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787828496.231895:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1787828496.231898:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787828496.231900:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828496.231904:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828496.231907:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787828496.231909:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1787828496.231912:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1787828496.231919:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828493/real 1787828496] req@ffff8d30d4d252c0 x1874670646735616/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828518 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1787828496.231933:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787828496.231935:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1787828496.231939:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1787828496.231942:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787828496.231944:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828496.231947:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828496.231949:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787828496.231955:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d252c0 x1874670646735616/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828518 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787828496.231976:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828496.231979:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787828496.231983:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828496.231985:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828496.231987:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828496.231990:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787828496.231993:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787828496.231996:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828496.232000:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:2.0:1787828496.232004:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1787828496.232008:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d252c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646735616:192.168.203.143@tcp:8:kworker.0 00000100:00000001:2.0:1787828496.232014:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828496.232016:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828496.232018:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787828496.232020:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1787828496.232024:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787828496.232026:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:2.0:1787828496.232027:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:2.0:1787828496.232030:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787828496.232032:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1787828496.232035:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787828496.232036:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828496.232040:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828496.232048:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1787828496.232051:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828496.232054:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787828496.232056:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828497.126754:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828497.126762:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787828497.126835:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787828497.126841:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787828497.126845:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828498.150707:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828498.150715:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787828498.150778:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787828498.150782:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787828498.150785:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787828498.278745:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828498.278759:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1787828498.278763:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787828498.278768:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787828498.278774:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787828498.278784:0:468:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 3298 00000100:00080000:3.0:1787828498.278798:0:468:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787828498.278802:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787828498.278805:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:3.0:1787828498.278812:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1787828498.278816:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1787828498.278821:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:3.0:1787828498.278826:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:3.0:1787828498.278829:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787828498.278831:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:3.0:1787828498.278838:0:468:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:3.0:1787828498.278843:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787828498.278847:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787828498.278851:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787828498.278856:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828498.278860:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:3.0:1787828498.278864:0:468:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787828498.278876:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787828498.278879:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828498.278884:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787828498.278888:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787828498.278894:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828498.278907:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787828498.278921:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c7016940 x1874670646735744/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787828498.278938:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787828498.278945:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828498.278972:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787828498.278977:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828498.278982:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1787828498.278985:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787828498.278988:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787828498.278992:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787828498.278997:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3298 00000100:00080000:3.0:1787828498.279003:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787828498.279007:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787828498.279010:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:3.0:1787828498.279026:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1787828498.279029:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1787828498.279033:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000020:00000001:3.0:1787828498.279037:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:3.0:1787828498.279040:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787828498.279043:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:2.0:1787828498.279051:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00080000:3.0:1787828498.279051:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:3.0:1787828498.279060:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828498.279061:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 02000000:00000001:3.0:1787828498.279064:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:2.0:1787828498.279068:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 02000000:00000001:3.0:1787828498.279075:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787828498.279076:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787828498.279083:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787828498.279084:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828498.279093:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 02000000:00000001:2.0:1787828498.279095:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000008:00100000:3.0:1787828498.279100:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 02000000:00000001:2.0:1787828498.279103:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000008:00000001:3.0:1787828498.279110:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828498.279113:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828498.279119:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787828498.279129:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c7016940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646735744:192.168.203.143@tcp:250:kworker.0 02000000:00000001:3.0:1787828498.279131:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787828498.279136:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828498.279142:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:3.0:1787828498.279145:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787828498.279150:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787828498.279152:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1787828498.279155:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828498.279160:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828498.279178:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787828498.279192:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c7017c00 x1874670646735872/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:2.0:1787828498.279207:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:3.0:1787828498.279208:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787828498.279216:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828498.279221:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828498.279223:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787828498.279232:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828498.279238:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1787828498.279241:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787828498.279244:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787828498.279249:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787828498.279249:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:2.0:1787828498.279258:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369546048 : -126234340005568 : ffff8d30c7016940) 00000100:00080000:3.0:1787828498.279260:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3298 00000100:00000001:2.0:1787828498.279269:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00080000:3.0:1787828498.279272:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787828498.279279:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787828498.279282:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:3.0:1787828498.279286:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1787828498.279289:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1787828498.279293:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000020:00000001:3.0:1787828498.279298:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:3.0:1787828498.279301:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787828498.279304:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:3.0:1787828498.279310:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:3.0:1787828498.279315:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787828498.279317:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787828498.279323:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000800:00000100:2.0:1787828498.279330:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 02000000:00000001:3.0:1787828498.279331:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828498.279336:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:3.0:1787828498.279340:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1787828498.279344:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828498.279347:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787828498.279353:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787828498.279357:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828498.279362:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787828498.279365:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787828498.279369:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828498.279374:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1787828498.279384:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828498.279387:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828498.279391:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828498.279392:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1787828498.279404:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787828498.279406:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c7014000 x1874670646736000/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787828498.279415:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4297972480 00000100:00000001:3.0:1787828498.279431:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787828498.279437:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828498.279438:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 00000100:00000001:3.0:1787828498.279445:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787828498.279451:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828498.279457:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1787828498.279459:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787828498.279462:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787828498.279467:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:2.0:1787828498.279472:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4297972480 00000100:00080000:3.0:1787828498.279473:0:468:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3298 00000100:00080000:3.0:1787828498.279484:0:468:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787828498.279489:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 00000100:00000001:3.0:1787828498.279491:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787828498.279496:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:2.0:1787828498.279502:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000020:00000001:3.0:1787828498.279503:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1787828498.279508:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:2.0:1787828498.279512:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828498.279514:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:2.0:1787828498.279521:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000020:00000001:3.0:1787828498.279532:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:3.0:1787828498.279536:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787828498.279539:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00100000:2.0:1787828498.279548:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828498/real 1787828498] req@ffff8d30c7016940 x1874670646735744/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828523 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828498.279549:0:468:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:3.0:1787828498.279558:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787828498.279565:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787828498.279571:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787828498.279579:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:2.0:1787828498.279581:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000001:3.0:1787828498.279584:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000400:00000001:2.0:1787828498.279651:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1787828498.279662:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1787828498.279670:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787828498.279674:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828498.279678:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828498.279681:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787828498.279691:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c7016940 x1874670646735744/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828523 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787828498.279714:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828498.279718:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787828498.279723:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828498.279725:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828498.279727:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828498.279730:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787828498.279733:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787828498.279738:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828498.279744:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:2.0:1787828498.279748:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1787828498.279754:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c7016940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646735744:192.168.203.143@tcp:250:kworker.0 00000100:00000001:2.0:1787828498.279761:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828498.279765:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828498.279767:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787828498.279771:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1787828498.279775:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787828498.279778:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:2.0:1787828498.279781:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:2.0:1787828498.279787:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787828498.279789:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1787828498.279793:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787828498.279796:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828498.279799:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828498.279808:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1787828498.279811:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828498.279815:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787828498.279818:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1787828498.279821:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1787828498.279823:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787828498.279825:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828498.279829:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787828498.279832:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828498.279835:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787828498.279842:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c7017c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646735872:192.168.203.143@tcp:38:kworker.0 00000100:00000001:2.0:1787828498.279848:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787828498.279852:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1787828498.279854:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828498.279857:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1787828498.279862:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000008:00100000:3.0:1787828498.279871:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00000001:2.0:1787828498.279874:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787828498.279879:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369550848 : -126234340000768 : ffff8d30c7017c00) 00000008:00000001:3.0:1787828498.279883:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828498.279891:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:3.0:1787828498.279891:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:2.0:1787828498.279925:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:2.0:1787828498.279935:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1787828498.279940:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828498.279942:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828498.279945:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1787828498.279952:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787828498.279955:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:2.0:1787828498.279960:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4297971223 02000000:00000001:3.0:1787828498.279961:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828498.279973:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4297972480 00000100:00000001:3.0:1787828498.279974:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787828498.279979:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:2.0:1787828498.279983:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4297971223 02000000:00000001:3.0:1787828498.279986:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828498.279992:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4297972480 00000100:00000001:2.0:1787828498.280000:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787828498.280002:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828498.280006:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828498.280013:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1787828498.280019:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c7017840 x1874670646736128/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787828498.280027:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828498/real 1787828498] req@ffff8d30c7017c00 x1874670646735872/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828523 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787828498.280052:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787828498.280061:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:2.0:1787828498.280062:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787828498.280068:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787828498.280071:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828498.280076:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1787828498.280084:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787828498.280087:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828498.280090:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828498.280094:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787828498.280103:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c7017c00 x1874670646735872/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828523 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787828498.280127:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828498.280132:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787828498.280136:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828498.280139:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828498.280142:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828498.280145:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787828498.280148:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787828498.280153:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828498.280159:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:2.0:1787828498.280163:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1787828498.280169:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c7017c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646735872:192.168.203.143@tcp:38:kworker.0 00000100:00000001:2.0:1787828498.280175:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1787828498.280177:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787828498.280179:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828498.280183:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787828498.280186:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828498.280189:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787828498.280194:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c7014000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646736000:192.168.203.143@tcp:38:kworker.0 00000100:00000001:2.0:1787828498.280199:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787828498.280203:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1787828498.280205:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828498.280208:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1787828498.280214:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828498.280223:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787828498.280225:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369535488 : -126234340016128 : ffff8d30c7014000) 00000100:00000001:2.0:1787828498.280231:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1787828498.280254:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:2.0:1787828498.280263:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1787828498.280270:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828498.280273:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828498.280277:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1787828498.280282:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828498.280288:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4297971223 00000100:00000001:2.0:1787828498.280293:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4297972480 00000100:00000001:2.0:1787828498.280296:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4297971223 00000100:00000001:2.0:1787828498.280299:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4297972480 00000100:00000001:2.0:1787828498.280302:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787828498.280306:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828498.280309:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1787828498.280316:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828498/real 1787828498] req@ffff8d30c7014000 x1874670646736000/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828523 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1787828498.280328:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787828498.280330:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1787828498.280334:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1787828498.280337:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787828498.280339:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828498.280342:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828498.280344:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787828498.280350:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c7014000 x1874670646736000/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828523 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787828498.280360:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828498.280364:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787828498.280367:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828498.280370:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828498.280372:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828498.280376:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787828498.280379:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787828498.280383:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828498.280391:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:2.0:1787828498.280395:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1787828498.280402:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c7014000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646736000:192.168.203.143@tcp:38:kworker.0 00000100:00000001:2.0:1787828498.280408:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828498.280411:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828498.280413:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787828498.280416:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1787828498.280419:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787828498.280421:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:2.0:1787828498.280424:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:2.0:1787828498.280438:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787828498.280440:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1787828498.280443:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787828498.280445:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828498.280448:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828498.280450:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787828498.280452:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1787828498.280455:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787828498.280457:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:2.0:1787828498.280458:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:2.0:1787828498.280461:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787828498.280464:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1787828498.280466:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787828498.280467:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828498.280470:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828498.280475:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1787828498.280477:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828498.280481:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787828498.280483:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1787828498.280486:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1787828498.280488:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787828498.280490:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828498.280505:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787828498.280508:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828498.280511:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787828498.280516:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c7017840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646736128:192.168.203.143@tcp:8:kworker.0 00000100:00000001:2.0:1787828498.280522:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787828498.280526:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1787828498.280528:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828498.280531:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1787828498.280536:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828498.280545:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787828498.280548:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369549888 : -126234340001728 : ffff8d30c7017840) 00000100:00000001:2.0:1787828498.280553:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1787828498.280574:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:2.0:1787828498.280582:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1787828498.280665:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828498.280668:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828498.280670:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1787828498.280676:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828498.280681:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4297971237 00000100:00000001:2.0:1787828498.280684:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4297972480 00000100:00000001:2.0:1787828498.280698:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4297971237 00000100:00000001:2.0:1787828498.280701:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4297972480 00000100:00000001:2.0:1787828498.280704:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787828498.280707:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828498.280710:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1787828498.280716:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828498/real 1787828498] req@ffff8d30c7017840 x1874670646736128/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828523 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1787828498.280728:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787828498.280730:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1787828498.280734:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1787828498.280736:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787828498.280738:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828498.280741:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828498.280743:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787828498.280748:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c7017840 x1874670646736128/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828523 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787828498.280757:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828498.280760:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787828498.280763:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828498.280765:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828498.280768:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828498.280770:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787828498.280773:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787828498.280776:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828498.280781:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:2.0:1787828498.280784:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1787828498.280789:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c7017840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646736128:192.168.203.143@tcp:8:kworker.0 00000100:00000001:2.0:1787828498.280805:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828498.280808:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828498.280810:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787828498.280812:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1787828498.280816:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787828498.280818:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:2.0:1787828498.280820:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:2.0:1787828498.280823:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787828498.280825:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1787828498.280828:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787828498.280829:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828498.280833:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828498.280837:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1787828498.280839:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828498.280842:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787828498.280845:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828499.174739:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828499.174746:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828499.174828:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828499.174834:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828499.174838:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828500.198742:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828500.198751:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787828500.198862:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787828500.198871:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787828500.198876:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828501.222724:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828501.222734:0:815:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1787828501.222737:0:815:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1787828501.222741:0:815:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828501.222748:0:815:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1787828501.222752:0:815:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828501.222756:0:815:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828501.222762:0:815:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787828501.222846:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787828501.222852:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787828501.222857:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828502.246699:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828502.246705:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828502.246769:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828502.246774:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828502.246779:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828503.270711:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828503.270718:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787828503.270795:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787828503.270799:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787828503.270802:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787828503.398725:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828503.398739:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1787828503.398742:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787828503.398748:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787828503.398754:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787828503.398769:0:40701:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 3303 00000100:00080000:2.0:1787828503.398781:0:40701:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787828503.398784:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828503.398787:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787828503.398790:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787828503.398793:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787828503.398796:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:2.0:1787828503.398799:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:2.0:1787828503.398801:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828503.398802:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:2.0:1787828503.398806:0:40701:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787828503.398808:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828503.398811:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787828503.398814:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828503.398817:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828503.398819:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:2.0:1787828503.398822:0:40701:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828503.398831:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787828503.398833:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828503.398836:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787828503.398839:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787828503.398843:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828503.398853:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787828503.398863:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2f0c0 x1874670646736256/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787828503.398897:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787828503.398901:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828503.398915:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787828503.398919:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828503.398923:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1787828503.398925:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787828503.398927:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787828503.398935:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787828503.398939:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:0.0:1787828503.398942:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00080000:2.0:1787828503.398946:0:40701:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3303 00000100:00000001:0.0:1787828503.398947:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1787828503.398951:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787828503.398954:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787828503.398956:0:40701:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787828503.398961:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 02000000:00000001:0.0:1787828503.398962:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:2.0:1787828503.398964:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 02000000:00000001:0.0:1787828503.398967:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000020:00000001:2.0:1787828503.398969:0:40701:0:(genops.c:792:class_conn2export()) Process entered 02000000:00000001:0.0:1787828503.398974:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1787828503.398975:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787828503.398979:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000020:00000001:2.0:1787828503.398985:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00100000:0.0:1787828503.398986:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646736256:192.168.203.143@tcp:250:kworker.0 00000100:00000001:2.0:1787828503.398988:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828503.398991:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:0.0:1787828503.398995:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00080000:2.0:1787828503.398999:0:40701:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 02000000:00000001:0.0:1787828503.399002:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:2.0:1787828503.399003:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828503.399008:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828503.399008:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787828503.399014:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1787828503.399016:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787828503.399020:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828503.399021:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828503.399027:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1787828503.399033:0:40701:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1787828503.399036:0:40701:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828503.399038:0:40701:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828503.399041:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 02000000:00000001:2.0:1787828503.399044:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:0.0:1787828503.399045:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417880768 : -126234291670848 : ffff8d30c9e2f0c0) 02000000:00000001:2.0:1787828503.399048:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828503.399051:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:2.0:1787828503.399053:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787828503.399056:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787828503.399058:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828503.399066:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787828503.399072:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2c3c0 x1874670646736384/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787828503.399079:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787828503.399082:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828503.399085:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787828503.399088:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828503.399092:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000800:00000100:0.0:1787828503.399095:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:2.0:1787828503.399096:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787828503.399101:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787828503.399104:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787828503.399109:0:40701:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3303 00000100:00000001:0.0:1787828503.399110:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00080000:2.0:1787828503.399114:0:40701:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787828503.399116:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828503.399118:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787828503.399121:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828503.399122:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:0.0:1787828503.399127:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000020:00000001:2.0:1787828503.399128:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787828503.399131:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787828503.399134:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:0.0:1787828503.399135:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1787828503.399137:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:2.0:1787828503.399141:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787828503.399143:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4297977600 00000100:00000001:2.0:1787828503.399144:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:2.0:1787828503.399152:0:40701:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787828503.399155:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828503.399156:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 02000000:00000001:2.0:1787828503.399157:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787828503.399163:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828503.399166:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828503.399168:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4297977600 00000100:00000001:2.0:1787828503.399169:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1787828503.399173:0:40701:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00000001:0.0:1787828503.399177:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 00000008:00000001:2.0:1787828503.399178:0:40701:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828503.399184:0:40701:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828503.399186:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787828503.399190:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828503.399191:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787828503.399195:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828503.399197:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00000001:2.0:1787828503.399199:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787828503.399202:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787828503.399204:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828503.399208:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828503/real 1787828503] req@ffff8d30c9e2f0c0 x1874670646736256/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828528 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787828503.399210:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787828503.399221:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2f480 x1874670646736512/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000001:0.0:1787828503.399228:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787828503.399232:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1787828503.399234:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787828503.399238:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1787828503.399239:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828503.399245:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787828503.399246:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828503.399250:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:2.0:1787828503.399251:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1787828503.399255:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00080000:2.0:1787828503.399257:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1787828503.399259:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828503.399261:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787828503.399264:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787828503.399267:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00100000:0.0:1787828503.399269:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2f0c0 x1874670646736256/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828528 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828503.399274:0:40701:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3303 00000100:00080000:2.0:1787828503.399282:0:40701:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787828503.399285:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828503.399287:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787828503.399291:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787828503.399292:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787828503.399298:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:0.0:1787828503.399299:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:2.0:1787828503.399300:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787828503.399305:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:0.0:1787828503.399306:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828503.399310:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:2.0:1787828503.399311:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:0.0:1787828503.399316:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787828503.399317:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787828503.399322:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787828503.399322:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:0.0:1787828503.399332:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828503.399333:0:40701:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787828503.399340:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787828503.399342:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 02000000:00000001:2.0:1787828503.399344:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:0.0:1787828503.399347:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 02000000:00000001:2.0:1787828503.399348:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828503.399353:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828503.399354:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646736256:192.168.203.143@tcp:250:kworker.0 00000100:00000001:2.0:1787828503.399356:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1787828503.399360:0:40701:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00000001:0.0:1787828503.399363:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000008:00000001:2.0:1787828503.399364:0:40701:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828503.399370:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828503.399371:0:40701:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828503.399375:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787828503.399378:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787828503.399380:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1787828503.399381:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828503.399386:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00000001:2.0:1787828503.399386:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000020:00000001:0.0:1787828503.399391:0:2390:0:(genops.c:1217:class_import_put()) Process entered 02000000:00000001:2.0:1787828503.399392:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000020:00000001:0.0:1787828503.399395:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:2.0:1787828503.399398:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828503.399400:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787828503.399404:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787828503.399408:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787828503.399410:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828503.399410:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828503.399434:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828503.399442:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00100000:2.0:1787828503.399442:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2c000 x1874670646736640/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787828503.399446:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828503.399451:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787828503.399454:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787828503.399457:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1787828503.399461:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787828503.399463:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787828503.399464:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828503.399470:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:2.0:1787828503.399471:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828503.399473:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787828503.399479:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828503.399483:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828503.399484:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646736384:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787828503.399492:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787828503.399494:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787828503.399496:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828503.399499:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1787828503.399502:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828503.399510:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787828503.399512:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417869248 : -126234291682368 : ffff8d30c9e2c3c0) 00000100:00000001:0.0:1787828503.399517:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1787828503.399531:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:0.0:1787828503.399539:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1787828503.399543:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828503.399545:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828503.399547:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787828503.399574:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828503.399581:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4297976342 00000100:00000001:0.0:1787828503.399585:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4297977600 00000100:00000001:0.0:1787828503.399588:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4297976342 00000100:00000001:0.0:1787828503.399591:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4297977600 00000100:00000001:0.0:1787828503.399594:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787828503.399596:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828503.399632:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787828503.399638:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828503/real 1787828503] req@ffff8d30c9e2c3c0 x1874670646736384/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828528 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787828503.399647:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787828503.399648:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787828503.399652:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787828503.399655:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787828503.399656:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828503.399658:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828503.399660:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787828503.399663:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2c3c0 x1874670646736384/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828528 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787828503.399670:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828503.399672:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787828503.399675:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828503.399677:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828503.399679:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828503.399680:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787828503.399683:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787828503.399686:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828503.399689:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787828503.399692:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787828503.399696:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646736384:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787828503.399700:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1787828503.399701:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787828503.399702:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828503.399704:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787828503.399706:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787828503.399708:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828503.399710:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646736512:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787828503.399712:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787828503.399714:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787828503.399716:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828503.399717:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1787828503.399720:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828503.399724:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787828503.399725:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417881728 : -126234291669888 : ffff8d30c9e2f480) 00000100:00000001:0.0:1787828503.399729:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1787828503.399739:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:0.0:1787828503.399743:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1787828503.399745:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828503.399746:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828503.399747:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787828503.399749:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828503.399752:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4297976343 00000100:00000001:0.0:1787828503.399754:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4297977600 00000100:00000001:0.0:1787828503.399756:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4297976343 00000100:00000001:0.0:1787828503.399758:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4297977600 00000100:00000001:0.0:1787828503.399760:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787828503.399762:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828503.399763:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787828503.399766:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828503/real 1787828503] req@ffff8d30c9e2f480 x1874670646736512/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828528 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787828503.399773:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787828503.399774:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787828503.399775:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787828503.399776:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787828503.399777:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828503.399779:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828503.399780:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787828503.399783:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2f480 x1874670646736512/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828528 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787828503.399789:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828503.399790:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787828503.399792:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828503.399793:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828503.399794:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828503.399795:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787828503.399797:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787828503.399798:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828503.399801:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787828503.399802:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787828503.399804:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646736512:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787828503.399807:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828503.399809:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828503.399810:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787828503.399811:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787828503.399812:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787828503.399813:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787828503.399815:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787828503.399816:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787828503.399818:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787828503.399820:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787828503.399821:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828503.399823:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828503.399824:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787828503.399825:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787828503.399826:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787828503.399827:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787828503.399828:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787828503.399829:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787828503.399830:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787828503.399831:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787828503.399832:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828503.399833:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828503.399836:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787828503.399837:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828503.399838:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787828503.399839:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787828503.399840:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1787828503.399841:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787828503.399842:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828503.399844:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787828503.399845:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787828503.399847:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828503.399849:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646736640:192.168.203.143@tcp:8:kworker.0 00000100:00000001:0.0:1787828503.399852:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787828503.399853:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787828503.399855:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828503.399856:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1787828503.399858:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828503.399862:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787828503.399863:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417868288 : -126234291683328 : ffff8d30c9e2c000) 00000100:00000001:0.0:1787828503.399865:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1787828503.399873:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:0.0:1787828503.399876:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1787828503.399878:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828503.399879:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828503.399880:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787828503.399882:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828503.399885:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4297976356 00000100:00000001:0.0:1787828503.399887:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4297977600 00000100:00000001:0.0:1787828503.399888:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4297976356 00000100:00000001:0.0:1787828503.399890:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4297977600 00000100:00000001:0.0:1787828503.399892:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787828503.399894:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828503.399895:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787828503.399898:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828503/real 1787828503] req@ffff8d30c9e2c000 x1874670646736640/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828528 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787828503.399904:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787828503.399905:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787828503.399906:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787828503.399907:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787828503.399908:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828503.399909:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828503.399910:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787828503.399913:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2c000 x1874670646736640/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828528 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787828503.399919:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828503.399920:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787828503.399922:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828503.399924:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828503.399925:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828503.399926:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787828503.399927:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787828503.399929:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828503.399931:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787828503.399933:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787828503.399935:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646736640:192.168.203.143@tcp:8:kworker.0 00000100:00000001:0.0:1787828503.399940:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828503.399941:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828503.399942:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787828503.399943:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787828503.399945:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787828503.399946:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787828503.399947:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787828503.399948:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787828503.399949:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787828503.399950:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787828503.399951:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828503.399952:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828503.399954:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787828503.399956:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828503.399957:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787828503.399958:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828504.294688:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828504.294698:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787828504.294769:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787828504.294774:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787828504.294778:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828505.318697:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828505.318705:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828505.318770:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828505.318775:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828505.318779:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828506.342728:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828506.342734:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787828506.342809:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787828506.342814:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787828506.342819:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828507.366735:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828507.366744:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787828507.366862:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787828507.366867:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787828507.366871:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828508.390706:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828508.390714:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828508.390767:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828508.390772:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828508.390776:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787828508.518746:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828508.518758:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1787828508.518761:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787828508.518765:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787828508.518771:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787828508.518780:0:468:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 3309 00000100:00080000:3.0:1787828508.518793:0:468:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787828508.518796:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787828508.518798:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:3.0:1787828508.518803:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1787828508.518807:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1787828508.518811:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:3.0:1787828508.518813:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:3.0:1787828508.518815:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787828508.518817:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:3.0:1787828508.518821:0:468:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:3.0:1787828508.518823:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787828508.518826:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787828508.518830:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787828508.518833:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828508.518836:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:3.0:1787828508.518839:0:468:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787828508.518855:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787828508.518857:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828508.518861:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787828508.518864:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787828508.518869:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828508.518884:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787828508.518894:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c70170c0 x1874670646736768/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787828508.518905:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787828508.518910:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828508.518928:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787828508.518931:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828508.518935:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1787828508.518936:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787828508.518938:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787828508.518941:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787828508.518943:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3309 00000100:00080000:3.0:1787828508.518945:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787828508.518947:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787828508.518949:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:3.0:1787828508.518951:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1787828508.518953:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1787828508.518955:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000020:00000001:3.0:1787828508.518958:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:3.0:1787828508.518959:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787828508.518961:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:3.0:1787828508.518964:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:3.0:1787828508.518966:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787828508.518967:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:0.0:1787828508.518970:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:3.0:1787828508.518974:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787828508.518978:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 02000000:00000001:3.0:1787828508.518979:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828508.518983:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787828508.518984:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1787828508.518989:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000008:00100000:3.0:1787828508.518992:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00000001:0.0:1787828508.518993:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:3.0:1787828508.518999:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828508.519003:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:3.0:1787828508.519003:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828508.519009:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787828508.519010:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787828508.519015:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828508.519016:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828508.519018:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787828508.519021:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787828508.519025:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828508.519027:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c70170c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646736768:192.168.203.143@tcp:250:kworker.0 00000100:00000001:3.0:1787828508.519031:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828508.519034:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00100000:3.0:1787828508.519040:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c7016d00 x1874670646736896/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:0.0:1787828508.519041:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787828508.519046:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828508.519052:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:3.0:1787828508.519057:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787828508.519059:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828508.519063:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828508.519063:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787828508.519073:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828508.519078:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1787828508.519081:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787828508.519083:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787828508.519087:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787828508.519091:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3309 00000100:00000001:0.0:1787828508.519092:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00080000:3.0:1787828508.519097:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787828508.519098:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369547968 : -126234340003648 : ffff8d30c70170c0) 00000100:00000001:3.0:1787828508.519104:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787828508.519109:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:3.0:1787828508.519110:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:3.0:1787828508.519117:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1787828508.519119:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1787828508.519122:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000020:00000001:3.0:1787828508.519125:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:3.0:1787828508.519129:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787828508.519130:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:3.0:1787828508.519136:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:3.0:1787828508.519140:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787828508.519142:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787828508.519145:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787828508.519147:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828508.519149:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:3.0:1787828508.519152:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1787828508.519154:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828508.519156:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787828508.519161:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787828508.519162:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828508.519164:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787828508.519166:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787828508.519170:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:0.0:1787828508.519172:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:3.0:1787828508.519178:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787828508.519187:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c7014b40 x1874670646737024/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787828508.519190:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1787828508.519198:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787828508.519198:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787828508.519203:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828508.519204:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828508.519207:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828508.519209:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00080000:3.0:1787828508.519211:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828508.519216:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1787828508.519218:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828508.519219:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787828508.519223:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787828508.519227:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:0.0:1787828508.519228:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4297982720 00000100:00080000:3.0:1787828508.519233:0:468:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3309 00000100:00080000:3.0:1787828508.519238:0:468:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787828508.519240:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787828508.519241:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:0.0:1787828508.519244:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 00000020:00000001:3.0:1787828508.519245:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1787828508.519248:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1787828508.519252:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000020:00000001:3.0:1787828508.519254:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:0.0:1787828508.519255:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4297982720 00000100:00000001:3.0:1787828508.519259:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787828508.519263:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:0.0:1787828508.519264:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 00000100:00080000:3.0:1787828508.519271:0:468:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787828508.519272:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787828508.519280:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828508.519281:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787828508.519288:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:0.0:1787828508.519289:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 02000000:00000001:3.0:1787828508.519291:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787828508.519295:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828508.519298:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00100000:0.0:1787828508.519300:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828508/real 1787828508] req@ffff8d30c70170c0 x1874670646736768/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828533 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1787828508.519301:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1787828508.519308:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828508.519313:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:0.0:1787828508.519321:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787828508.519323:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787828508.519326:0:2390:0:(events.c:78:reply_in_callback()) Process entered 02000000:00000001:3.0:1787828508.519331:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:0.0:1787828508.519332:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 02000000:00000001:3.0:1787828508.519337:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828508.519338:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787828508.519342:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:0.0:1787828508.519345:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 02000000:00000001:3.0:1787828508.519345:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787828508.519350:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828508.519352:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787828508.519355:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828508.519368:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c70170c0 x1874670646736768/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828533 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828508.519368:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c70152c0 x1874670646737152/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787828508.519401:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787828508.519406:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828508.519407:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787828508.519410:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828508.519415:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787828508.519420:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828508.519422:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828508.519423:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828508.519426:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787828508.519429:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787828508.519432:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828508.519437:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787828508.519440:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787828508.519445:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c70170c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646736768:192.168.203.143@tcp:250:kworker.0 00000100:00000001:0.0:1787828508.519454:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828508.519456:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828508.519458:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787828508.519461:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787828508.519466:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787828508.519468:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787828508.519470:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787828508.519475:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787828508.519477:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787828508.519479:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787828508.519481:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828508.519483:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828508.519493:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787828508.519495:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828508.519497:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787828508.519498:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787828508.519500:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1787828508.519501:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787828508.519502:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828508.519504:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787828508.519506:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787828508.519508:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828508.519511:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c7016d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646736896:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787828508.519533:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787828508.519536:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787828508.519538:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828508.519539:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1787828508.519543:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828508.519549:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787828508.519551:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369547008 : -126234340004608 : ffff8d30c7016d00) 00000100:00000001:0.0:1787828508.519556:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1787828508.519569:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:0.0:1787828508.519574:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1787828508.519576:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828508.519578:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828508.519579:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787828508.519582:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828508.519587:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4297981462 00000100:00000001:0.0:1787828508.519590:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4297982720 00000100:00000001:0.0:1787828508.519593:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4297981462 00000100:00000001:0.0:1787828508.519595:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4297982720 00000100:00000001:0.0:1787828508.519597:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787828508.519625:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828508.519627:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787828508.519632:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828508/real 1787828508] req@ffff8d30c7016d00 x1874670646736896/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828533 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787828508.519640:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787828508.519641:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787828508.519643:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787828508.519645:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787828508.519646:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828508.519648:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828508.519649:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787828508.519652:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c7016d00 x1874670646736896/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828533 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787828508.519658:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828508.519660:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787828508.519662:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828508.519663:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828508.519664:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828508.519666:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787828508.519668:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787828508.519670:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828508.519673:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787828508.519676:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787828508.519678:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c7016d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646736896:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787828508.519684:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1787828508.519686:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787828508.519687:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828508.519689:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787828508.519691:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787828508.519693:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828508.519695:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c7014b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646737024:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787828508.519698:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787828508.519700:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787828508.519701:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828508.519703:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1787828508.519706:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828508.519710:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787828508.519712:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369538368 : -126234340013248 : ffff8d30c7014b40) 00000100:00000001:0.0:1787828508.519715:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1787828508.519725:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:0.0:1787828508.519728:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1787828508.519730:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828508.519732:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828508.519733:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787828508.519735:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828508.519738:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4297981463 00000100:00000001:0.0:1787828508.519741:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4297982720 00000100:00000001:0.0:1787828508.519743:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4297981463 00000100:00000001:0.0:1787828508.519745:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4297982720 00000100:00000001:0.0:1787828508.519747:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787828508.519748:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828508.519750:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787828508.519753:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828508/real 1787828508] req@ffff8d30c7014b40 x1874670646737024/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828533 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787828508.519759:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787828508.519761:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787828508.519762:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787828508.519764:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787828508.519765:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828508.519767:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828508.519768:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787828508.519771:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c7014b40 x1874670646737024/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828533 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787828508.519776:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828508.519778:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787828508.519780:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828508.519781:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828508.519782:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828508.519783:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787828508.519784:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787828508.519786:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828508.519789:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787828508.519790:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787828508.519793:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c7014b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646737024:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787828508.519795:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1787828508.519796:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787828508.519798:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828508.519799:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787828508.519800:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787828508.519802:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828508.519804:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c70152c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646737152:192.168.203.143@tcp:8:kworker.0 00000100:00000001:0.0:1787828508.519807:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787828508.519808:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787828508.519809:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828508.519811:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1787828508.519813:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828508.519818:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787828508.519820:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369540288 : -126234340011328 : ffff8d30c70152c0) 00000100:00000001:0.0:1787828508.519823:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1787828508.519835:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:0.0:1787828508.519840:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1787828508.519843:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828508.519845:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828508.519847:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787828508.519850:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828508.519854:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4297981476 00000100:00000001:0.0:1787828508.519858:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4297982720 00000100:00000001:0.0:1787828508.519861:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4297981476 00000100:00000001:0.0:1787828508.519864:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4297982720 00000100:00000001:0.0:1787828508.519867:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787828508.519870:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828508.519873:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787828508.519879:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828508/real 1787828508] req@ffff8d30c70152c0 x1874670646737152/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828533 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787828508.519891:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787828508.519892:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787828508.519896:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787828508.519898:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787828508.519900:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828508.519902:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828508.519904:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787828508.519907:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c70152c0 x1874670646737152/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828533 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787828508.519914:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828508.519916:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787828508.519918:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828508.519920:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828508.519921:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828508.519922:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787828508.519924:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787828508.519927:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828508.519929:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787828508.519932:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787828508.519934:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c70152c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646737152:192.168.203.143@tcp:8:kworker.0 00000100:00000001:0.0:1787828508.519937:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828508.519939:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828508.519940:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787828508.519941:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787828508.519943:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787828508.519944:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787828508.519946:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787828508.519948:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787828508.519949:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787828508.519951:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787828508.519953:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828508.519954:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828508.519955:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787828508.519956:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787828508.519958:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787828508.519959:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787828508.519960:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787828508.519961:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787828508.519962:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787828508.519964:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787828508.519965:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828508.519966:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828508.519967:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787828508.519968:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787828508.519969:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787828508.519970:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787828508.519971:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787828508.519972:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787828508.519973:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787828508.519974:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787828508.519975:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828508.519977:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828508.519979:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787828508.519981:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828508.519983:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787828508.519985:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828509.414724:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828509.414731:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787828509.414849:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787828509.414862:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787828509.414872:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828510.438686:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828510.438693:0:815:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1787828510.438695:0:815:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1787828510.438698:0:815:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828510.438703:0:815:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1787828510.438705:0:815:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828510.438707:0:815:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828510.438710:0:815:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787828510.438777:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787828510.438781:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787828510.438785:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828511.462717:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828511.462724:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828511.462809:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828511.462815:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828511.462819:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828512.486726:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828512.486735:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787828512.486818:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787828512.486824:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787828512.486829:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828513.510735:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828513.510744:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787828513.510851:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787828513.510862:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787828513.510871:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787828513.638778:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828513.638799:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1787828513.638805:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787828513.638814:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787828513.638823:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787828513.638837:0:468:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 3314 00000100:00080000:0.0:1787828513.638858:0:468:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787828513.638862:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787828513.638865:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787828513.638872:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787828513.638891:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787828513.638899:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:0.0:1787828513.638905:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:0.0:1787828513.638909:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787828513.638912:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:0.0:1787828513.638923:0:468:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787828513.638929:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828513.638935:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787828513.638943:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787828513.638949:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828513.638954:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1787828513.638961:0:468:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828513.638985:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787828513.638988:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828513.638993:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787828513.638997:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787828513.639005:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828513.639028:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828513.639048:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d270c0 x1874670646737280/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787828513.639087:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787828513.639097:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828513.639137:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787828513.639144:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828513.639150:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1787828513.639154:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787828513.639158:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787828513.639164:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787828513.639170:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3314 00000100:00080000:0.0:1787828513.639179:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787828513.639185:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787828513.639188:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787828513.639191:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787828513.639193:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787828513.639211:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000020:00000001:0.0:1787828513.639217:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:0.0:1787828513.639223:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787828513.639227:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:0.0:1787828513.639236:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787828513.639241:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787828513.639244:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828513.639252:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 02000000:00000001:0.0:1787828513.639253:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:2.0:1787828513.639258:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1787828513.639263:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787828513.639266:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828513.639270:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828513.639276:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787828513.639280:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828513.639282:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828513.639290:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828513.639291:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1787828513.639303:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828513.639309:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d270c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646737280:192.168.203.143@tcp:250:kworker.0 00000100:00000001:2.0:1787828513.639320:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000008:00000001:0.0:1787828513.639326:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828513.639330:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:0.0:1787828513.639333:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828513.639337:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828513.639345:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787828513.639347:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1787828513.639356:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828513.639361:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787828513.639365:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828513.639366:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787828513.639377:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828513.639389:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828513.639394:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787828513.639398:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601348800 : -126234108202816 : ffff8d30d4d270c0) 00000100:00100000:0.0:1787828513.639403:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d24000 x1874670646737408/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787828513.639408:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:0.0:1787828513.639436:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787828513.639447:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828513.639456:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787828513.639460:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828513.639466:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1787828513.639468:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787828513.639471:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787828513.639475:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787828513.639504:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3314 00000100:00080000:0.0:1787828513.639511:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787828513.639525:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787828513.639527:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787828513.639535:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000800:00000100:2.0:1787828513.639538:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000020:00000001:0.0:1787828513.639542:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787828513.639550:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000020:00000001:0.0:1787828513.639557:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:0.0:1787828513.639562:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828513.639563:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1787828513.639567:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:0.0:1787828513.639592:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787828513.639594:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828513.639602:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828513.639606:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1787828513.639621:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828513.639630:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4297987841 00000100:00000001:0.0:1787828513.639661:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828513.639665:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787828513.639671:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787828513.639675:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828513.639679:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1787828513.639683:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1787828513.639688:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828513.639695:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828513.639705:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:2.0:1787828513.639706:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 02000000:00000001:0.0:1787828513.639709:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828513.639716:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787828513.639719:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787828513.639724:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828513.639724:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4297987841 00000100:00000001:0.0:1787828513.639735:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828513.639757:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 00000100:00100000:0.0:1787828513.639766:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d24f00 x1874670646737536/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787828513.639770:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787828513.639782:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828513.639793:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00000001:0.0:1787828513.639802:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787828513.639811:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787828513.639816:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828513/real 1787828513] req@ffff8d30d4d270c0 x1874670646737280/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828538 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787828513.639820:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787828513.639834:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828513.639851:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1787828513.639864:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000400:00000001:2.0:1787828513.639868:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00080000:0.0:1787828513.639869:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000400:00000001:2.0:1787828513.639876:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787828513.639878:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:2.0:1787828513.639888:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00080000:0.0:1787828513.639890:0:468:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3314 00000100:00000001:2.0:1787828513.639894:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787828513.639902:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828513.639906:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00080000:0.0:1787828513.639907:0:468:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787828513.639910:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828513.639922:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00100000:2.0:1787828513.639929:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d270c0 x1874670646737280/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828538 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787828513.639930:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787828513.639949:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787828513.639958:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:2.0:1787828513.639962:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1787828513.639967:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:2.0:1787828513.639969:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787828513.639985:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1787828513.639986:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:2.0:1787828513.639992:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828513.639995:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828513.639996:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828513.640008:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787828513.640012:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:2.0:1787828513.640013:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787828513.640029:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828513.640041:0:468:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00080000:2.0:1787828513.640042:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:2.0:1787828513.640050:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000001:0.0:1787828513.640065:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787828513.640066:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d270c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646737280:192.168.203.143@tcp:250:kworker.0 02000000:00000001:0.0:1787828513.640083:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:2.0:1787828513.640085:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828513.640096:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828513.640099:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787828513.640101:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828513.640103:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1787828513.640109:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 02000000:00000001:0.0:1787828513.640113:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1787828513.640113:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:2.0:1787828513.640119:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 00000100:00000001:0.0:1787828513.640127:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 02000000:00000001:2.0:1787828513.640128:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787828513.640134:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000008:00100000:0.0:1787828513.640140:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00000001:2.0:1787828513.640141:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787828513.640147:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828513.640162:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000008:00000001:0.0:1787828513.640165:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828513.640176:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828513.640177:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1787828513.640184:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828513.640192:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:0.0:1787828513.640193:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:2.0:1787828513.640200:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 02000000:00000001:0.0:1787828513.640201:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828513.640209:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1787828513.640214:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:2.0:1787828513.640214:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787828513.640226:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828513.640228:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787828513.640230:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787828513.640237:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787828513.640240:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828513.640242:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787828513.640261:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d24000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646737408:192.168.203.143@tcp:38:kworker.0 00000100:00000001:2.0:1787828513.640267:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:0.0:1787828513.640268:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828513.640271:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1787828513.640277:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828513.640281:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1787828513.640285:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828513.640294:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d25a40 x1874670646737664/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787828513.640297:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787828513.640302:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601336320 : -126234108215296 : ffff8d30d4d24000) 00000100:00000001:2.0:1787828513.640322:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:0.0:1787828513.640332:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787828513.640338:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828513.640347:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:2.0:1787828513.640356:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:2.0:1787828513.640368:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1787828513.640373:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828513.640376:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828513.640380:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1787828513.640389:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828513.640394:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4297986583 00000100:00000001:2.0:1787828513.640398:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4297987841 00000100:00000001:2.0:1787828513.640401:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4297986583 00000100:00000001:2.0:1787828513.640403:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4297987841 00000100:00000001:2.0:1787828513.640405:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787828513.640408:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828513.640410:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1787828513.640415:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828513/real 1787828513] req@ffff8d30d4d24000 x1874670646737408/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828538 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1787828513.640425:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787828513.640427:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1787828513.640430:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1787828513.640431:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787828513.640433:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828513.640435:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828513.640437:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787828513.640441:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d24000 x1874670646737408/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828538 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787828513.640448:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828513.640450:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787828513.640453:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828513.640455:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828513.640456:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828513.640458:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787828513.640460:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787828513.640462:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828513.640467:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:2.0:1787828513.640470:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1787828513.640475:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d24000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646737408:192.168.203.143@tcp:38:kworker.0 00000100:00000001:2.0:1787828513.640505:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1787828513.640508:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787828513.640510:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828513.640513:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787828513.640517:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828513.640536:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787828513.640542:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d24f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646737536:192.168.203.143@tcp:38:kworker.0 00000100:00000001:2.0:1787828513.640549:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787828513.640553:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1787828513.640556:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828513.640560:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1787828513.640566:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828513.640576:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787828513.640579:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601340160 : -126234108211456 : ffff8d30d4d24f00) 00000100:00000001:2.0:1787828513.640586:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1787828513.640608:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:2.0:1787828513.640644:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1787828513.640648:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828513.640650:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828513.640652:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1787828513.640656:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828513.640661:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4297986584 00000100:00000001:2.0:1787828513.640665:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4297987841 00000100:00000001:2.0:1787828513.640667:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4297986584 00000100:00000001:2.0:1787828513.640670:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4297987841 00000100:00000001:2.0:1787828513.640673:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787828513.640675:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828513.640677:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1787828513.640683:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828513/real 1787828513] req@ffff8d30d4d24f00 x1874670646737536/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828538 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1787828513.640692:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787828513.640693:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1787828513.640696:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1787828513.640698:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787828513.640700:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828513.640702:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828513.640704:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787828513.640708:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d24f00 x1874670646737536/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828538 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787828513.640716:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828513.640719:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787828513.640722:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828513.640724:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828513.640726:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828513.640728:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787828513.640753:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787828513.640757:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828513.640763:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:2.0:1787828513.640767:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1787828513.640772:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d24f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646737536:192.168.203.143@tcp:38:kworker.0 00000100:00000001:2.0:1787828513.640778:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828513.640782:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828513.640784:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787828513.640786:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1787828513.640791:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787828513.640793:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:2.0:1787828513.640796:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:2.0:1787828513.640800:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787828513.640802:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1787828513.640807:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787828513.640809:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828513.640812:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828513.640815:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787828513.640817:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1787828513.640819:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787828513.640821:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:2.0:1787828513.640824:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:2.0:1787828513.640827:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787828513.640829:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1787828513.640833:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787828513.640835:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828513.640839:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828513.640845:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1787828513.640848:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828513.640850:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787828513.640853:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1787828513.640854:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1787828513.640856:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787828513.640859:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828513.640862:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787828513.640864:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828513.640866:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787828513.640870:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d25a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646737664:192.168.203.143@tcp:8:kworker.0 00000100:00000001:2.0:1787828513.640875:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787828513.640877:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1787828513.640879:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828513.640881:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1787828513.640885:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828513.640892:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787828513.640894:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601343040 : -126234108208576 : ffff8d30d4d25a40) 00000100:00000001:2.0:1787828513.640897:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1787828513.640913:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:2.0:1787828513.640918:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1787828513.640921:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828513.640922:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828513.640924:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1787828513.640928:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828513.640932:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4297986597 00000100:00000001:2.0:1787828513.640934:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4297987841 00000100:00000001:2.0:1787828513.640937:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4297986597 00000100:00000001:2.0:1787828513.640939:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4297987841 00000100:00000001:2.0:1787828513.640942:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787828513.640944:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828513.640946:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1787828513.640950:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828513/real 1787828513] req@ffff8d30d4d25a40 x1874670646737664/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828538 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1787828513.640958:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787828513.640959:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1787828513.640962:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1787828513.640964:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787828513.640965:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828513.640968:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828513.640970:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787828513.640974:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d25a40 x1874670646737664/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828538 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787828513.640982:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828513.640984:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787828513.640986:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828513.640988:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828513.640989:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828513.640991:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787828513.640993:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787828513.640995:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828513.640999:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:2.0:1787828513.641001:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1787828513.641004:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d25a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646737664:192.168.203.143@tcp:8:kworker.0 00000100:00000001:2.0:1787828513.641007:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828513.641009:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828513.641011:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787828513.641012:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1787828513.641014:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787828513.641016:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:2.0:1787828513.641017:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:2.0:1787828513.641020:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787828513.641021:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1787828513.641024:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787828513.641025:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828513.641027:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828513.641030:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1787828513.641032:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828513.641034:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787828513.641036:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828514.534684:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828514.534690:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828514.534750:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828514.534753:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828514.534756:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828515.558670:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828515.558676:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787828515.558720:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787828515.558723:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787828515.558726:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828516.582717:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828516.582727:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787828516.582830:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787828516.582836:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787828516.582841:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828517.606699:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828517.606706:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828517.606775:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828517.606780:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828517.606784:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828518.630753:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828518.630762:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787828518.630936:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787828518.630942:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787828518.630947:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787828518.758755:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828518.758767:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1787828518.758780:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787828518.758801:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787828518.758807:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787828518.758816:0:40701:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 3319 00000100:00080000:2.0:1787828518.758829:0:40701:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787828518.758833:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828518.758835:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787828518.758840:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787828518.758844:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787828518.758848:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:2.0:1787828518.758850:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:2.0:1787828518.758853:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828518.758855:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:2.0:1787828518.758861:0:40701:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787828518.758864:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828518.758884:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787828518.758887:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828518.758890:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828518.758892:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:2.0:1787828518.758895:0:40701:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828518.758908:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787828518.758909:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828518.758912:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787828518.758915:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787828518.758921:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828518.758934:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787828518.758974:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2da40 x1874670646737792/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787828518.758988:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787828518.758993:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828518.759016:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787828518.759021:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828518.759026:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1787828518.759028:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787828518.759030:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787828518.759033:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787828518.759036:0:40701:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3319 00000100:00080000:2.0:1787828518.759041:0:40701:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787828518.759043:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828518.759045:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787828518.759060:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787828518.759062:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787828518.759065:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000020:00000001:2.0:1787828518.759068:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:2.0:1787828518.759076:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828518.759078:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:2.0:1787828518.759082:0:40701:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787828518.759095:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828518.759098:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787828518.759105:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828518.759109:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828518.759111:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1787828518.759117:0:40701:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1787828518.759121:0:40701:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828518.759124:0:40701:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828518.759130:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:2.0:1787828518.759131:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787828518.759134:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828518.759140:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:0.0:1787828518.759143:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 02000000:00000001:2.0:1787828518.759143:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787828518.759152:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828518.759153:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1787828518.759161:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828518.759162:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787828518.759172:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787828518.759176:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2ed00 x1874670646737920/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:0.0:1787828518.759184:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787828518.759195:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787828518.759195:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787828518.759203:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828518.759209:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828518.759211:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787828518.759236:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1787828518.759241:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2da40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646737792:192.168.203.143@tcp:250:kworker.0 00000100:00080000:2.0:1787828518.759243:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1787828518.759250:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:0.0:1787828518.759257:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00080000:2.0:1787828518.759257:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787828518.759266:0:40701:0:(import.c:485:import_select_connection()) Process entered 02000000:00000001:0.0:1787828518.759267:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787828518.759275:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787828518.759276:0:40701:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3319 02000000:00000001:0.0:1787828518.759283:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00080000:2.0:1787828518.759287:0:40701:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787828518.759294:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 02000000:00000001:0.0:1787828518.759295:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828518.759297:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787828518.759305:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787828518.759308:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787828518.759312:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000020:00000001:2.0:1787828518.759316:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:2.0:1787828518.759320:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787828518.759321:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787828518.759323:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:0.0:1787828518.759325:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417875008 : -126234291676608 : ffff8d30c9e2da40) 00000100:00000001:0.0:1787828518.759336:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00080000:2.0:1787828518.759341:0:40701:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787828518.759347:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828518.759358:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787828518.759380:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828518.759384:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828518.759388:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1787828518.759391:0:40701:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1787828518.759396:0:40701:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828518.759401:0:40701:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828518.759412:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787828518.759417:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:0.0:1787828518.759419:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:2.0:1787828518.759427:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787828518.759470:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787828518.759475:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828518.759496:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1787828518.759497:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828518.759504:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828518.759509:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828518.759514:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00100000:2.0:1787828518.759516:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2c000 x1874670646738048/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787828518.759521:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828518.759539:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787828518.759543:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4297992960 00000100:00000001:2.0:1787828518.759549:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828518.759568:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828518.759572:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 00000100:00080000:2.0:1787828518.759574:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1787828518.759583:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4297992960 00000100:00080000:2.0:1787828518.759584:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1787828518.759592:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:0.0:1787828518.759594:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 00000100:00080000:2.0:1787828518.759596:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787828518.759641:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:0.0:1787828518.759645:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00080000:2.0:1787828518.759648:0:40701:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3319 00000100:00000001:0.0:1787828518.759653:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787828518.759659:0:40701:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787828518.759660:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00000001:2.0:1787828518.759668:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828518.759674:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00100000:0.0:1787828518.759675:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828518/real 1787828518] req@ffff8d30c9e2da40 x1874670646737792/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828543 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:2.0:1787828518.759683:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787828518.759705:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787828518.759709:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000020:00000001:2.0:1787828518.759714:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000400:00000001:0.0:1787828518.759716:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000001:2.0:1787828518.759717:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000001:0.0:1787828518.759722:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1787828518.759723:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:0.0:1787828518.759730:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00080000:2.0:1787828518.759732:0:40701:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787828518.759738:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787828518.759739:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828518.759743:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:2.0:1787828518.759744:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:0.0:1787828518.759751:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 02000000:00000001:2.0:1787828518.759769:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787828518.759771:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:2.0:1787828518.759782:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828518.759790:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00100000:0.0:1787828518.759791:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2da40 x1874670646737792/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828543 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:2.0:1787828518.759794:0:40701:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000008:00000001:2.0:1787828518.759805:0:40701:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828518.759812:0:40701:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828518.759823:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:2.0:1787828518.759826:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:0.0:1787828518.759829:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 02000000:00000001:2.0:1787828518.759832:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828518.759839:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828518.759840:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787828518.759846:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:0.0:1787828518.759847:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 02000000:00000001:2.0:1787828518.759851:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828518.759853:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828518.759860:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787828518.759863:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787828518.759864:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787828518.759888:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:2.0:1787828518.759896:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2f480 x1874670646738176/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828518.759897:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787828518.759905:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787828518.759915:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2da40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646737792:192.168.203.143@tcp:250:kworker.0 00000100:00000001:2.0:1787828518.759919:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787828518.759923:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828518.759924:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828518.759950:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828518.759951:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828518.759962:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787828518.759969:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787828518.759973:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787828518.759976:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787828518.759978:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787828518.759984:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787828518.759987:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787828518.759992:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787828518.759994:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828518.759998:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828518.760008:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787828518.760010:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828518.760013:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787828518.760015:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787828518.760017:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1787828518.760019:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787828518.760021:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828518.760024:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787828518.760027:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787828518.760030:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828518.760034:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646737920:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787828518.760039:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787828518.760042:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787828518.760044:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828518.760046:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1787828518.760050:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828518.760058:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787828518.760060:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417879808 : -126234291671808 : ffff8d30c9e2ed00) 00000100:00000001:0.0:1787828518.760078:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1787828518.760103:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:0.0:1787828518.760112:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1787828518.760117:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828518.760120:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828518.760124:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787828518.760129:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828518.760138:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4297991703 00000100:00000001:0.0:1787828518.760144:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4297992960 00000100:00000001:0.0:1787828518.760148:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4297991703 00000100:00000001:0.0:1787828518.760153:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4297992960 00000100:00000001:0.0:1787828518.760157:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787828518.760162:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828518.760166:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787828518.760175:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828518/real 1787828518] req@ffff8d30c9e2ed00 x1874670646737920/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828543 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787828518.760193:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787828518.760195:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787828518.760199:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787828518.760201:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787828518.760203:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828518.760205:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828518.760207:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787828518.760213:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2ed00 x1874670646737920/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828543 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787828518.760238:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828518.760241:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787828518.760244:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828518.760247:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828518.760249:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828518.760252:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787828518.760255:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787828518.760259:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828518.760264:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787828518.760268:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787828518.760275:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646737920:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787828518.760281:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1787828518.760283:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787828518.760285:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828518.760289:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787828518.760292:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787828518.760296:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828518.760301:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646738048:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787828518.760306:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787828518.760310:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787828518.760312:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828518.760315:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1787828518.760320:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828518.760326:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787828518.760328:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417868288 : -126234291683328 : ffff8d30c9e2c000) 00000100:00000001:0.0:1787828518.760332:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1787828518.760346:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:0.0:1787828518.760379:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1787828518.760382:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828518.760384:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828518.760387:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787828518.760390:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828518.760395:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4297991703 00000100:00000001:0.0:1787828518.760398:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4297992960 00000100:00000001:0.0:1787828518.760401:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4297991703 00000100:00000001:0.0:1787828518.760404:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4297992960 00000100:00000001:0.0:1787828518.760407:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787828518.760409:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828518.760412:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787828518.760417:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828518/real 1787828518] req@ffff8d30c9e2c000 x1874670646738048/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828543 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787828518.760477:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787828518.760479:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787828518.760482:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787828518.760484:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787828518.760486:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828518.760488:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828518.760500:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787828518.760506:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2c000 x1874670646738048/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828543 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787828518.760514:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828518.760517:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787828518.760520:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828518.760521:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828518.760523:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828518.760525:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787828518.760527:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787828518.760529:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828518.760533:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787828518.760535:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787828518.760539:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646738048:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787828518.760543:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1787828518.760544:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787828518.760546:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828518.760549:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787828518.760551:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787828518.760553:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828518.760568:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646738176:192.168.203.143@tcp:8:kworker.0 00000100:00000001:0.0:1787828518.760571:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787828518.760573:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787828518.760576:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828518.760578:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1787828518.760582:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828518.760588:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787828518.760590:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417881728 : -126234291669888 : ffff8d30c9e2f480) 00000100:00000001:0.0:1787828518.760594:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1787828518.760635:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:0.0:1787828518.760641:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1787828518.760643:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828518.760645:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828518.760647:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787828518.760650:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828518.760654:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4297991717 00000100:00000001:0.0:1787828518.760657:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4297992960 00000100:00000001:0.0:1787828518.760660:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4297991717 00000100:00000001:0.0:1787828518.760663:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4297992960 00000100:00000001:0.0:1787828518.760665:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787828518.760667:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828518.760669:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787828518.760674:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828518/real 1787828518] req@ffff8d30c9e2f480 x1874670646738176/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828543 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787828518.760682:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787828518.760683:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787828518.760708:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787828518.760710:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787828518.760711:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828518.760713:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828518.760714:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787828518.760718:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2f480 x1874670646738176/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828543 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787828518.760725:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828518.760727:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787828518.760730:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828518.760732:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828518.760733:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828518.760735:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787828518.760737:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787828518.760739:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828518.760742:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787828518.760744:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787828518.760747:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646738176:192.168.203.143@tcp:8:kworker.0 00000100:00000001:0.0:1787828518.760750:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828518.760752:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828518.760753:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787828518.760755:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787828518.760778:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787828518.760781:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787828518.760782:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787828518.760785:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787828518.760786:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787828518.760792:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787828518.760793:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828518.760795:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828518.760796:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787828518.760798:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787828518.760800:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787828518.760802:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787828518.760803:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787828518.760805:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787828518.760806:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787828518.760809:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787828518.760811:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828518.760812:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828518.760813:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787828518.760815:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787828518.760816:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787828518.760817:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787828518.760819:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787828518.760820:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787828518.760822:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787828518.760824:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787828518.760825:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828518.760827:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828518.760849:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787828518.760851:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828518.760853:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787828518.760856:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828519.654679:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828519.654684:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787828519.654731:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787828519.654734:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787828519.654737:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828520.678787:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828520.678799:0:815:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1787828520.678802:0:815:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1787828520.678805:0:815:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828520.678812:0:815:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1787828520.678814:0:815:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828520.678817:0:815:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828520.678820:0:815:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828520.678902:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828520.678908:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828520.678913:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828521.702674:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828521.702679:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787828521.702746:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787828521.702749:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787828521.702752:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828522.726753:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828522.726762:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787828522.726897:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787828522.726902:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787828522.726907:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828523.750709:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828523.750717:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828523.750795:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828523.750802:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828523.750808:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787828523.878710:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828523.878720:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1787828523.878722:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787828523.878726:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787828523.878730:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787828523.878737:0:40701:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 3324 00000100:00080000:2.0:1787828523.878747:0:40701:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787828523.878749:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828523.878752:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787828523.878755:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787828523.878757:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787828523.878761:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:2.0:1787828523.878763:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:2.0:1787828523.878764:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828523.878766:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:2.0:1787828523.878770:0:40701:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787828523.878772:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828523.878774:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787828523.878777:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828523.878779:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828523.878781:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:2.0:1787828523.878784:0:40701:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828523.878794:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787828523.878796:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828523.878799:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787828523.878811:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787828523.878815:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828523.878825:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787828523.878834:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2c3c0 x1874670646738304/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787828523.878844:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787828523.878847:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828523.878861:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787828523.878864:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828523.878867:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1787828523.878868:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787828523.878870:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787828523.878872:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787828523.878874:0:40701:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3324 00000100:00080000:2.0:1787828523.878877:0:40701:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787828523.878878:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828523.878879:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787828523.878882:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787828523.878883:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787828523.878885:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000020:00000001:2.0:1787828523.878886:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:2.0:1787828523.878888:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828523.878889:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:2.0:1787828523.878891:0:40701:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787828523.878893:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828523.878894:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787828523.878898:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828523.878900:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828523.878903:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1787828523.878924:0:40701:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1787828523.878927:0:40701:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828523.878929:0:40701:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828523.878932:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787828523.878933:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828523.878934:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787828523.878936:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787828523.878937:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828523.878941:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787828523.878946:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2f0c0 x1874670646738432/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787828523.878958:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787828523.878960:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828523.878962:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787828523.878965:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828523.878967:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1787828523.878968:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787828523.878969:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787828523.878971:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787828523.878973:0:40701:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3324 00000100:00000001:0.0:1787828523.878976:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00080000:2.0:1787828523.878977:0:40701:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787828523.878987:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828523.878990:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:0.0:1787828523.878992:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000020:00000001:2.0:1787828523.878994:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:0.0:1787828523.879000:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000020:00000001:2.0:1787828523.879002:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787828523.879014:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:0.0:1787828523.879016:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787828523.879021:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1787828523.879022:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:2.0:1787828523.879034:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 02000000:00000001:0.0:1787828523.879038:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:2.0:1787828523.879044:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 02000000:00000001:0.0:1787828523.879046:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787828523.879057:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787828523.879062:0:40701:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787828523.879069:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828523.879076:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646738304:192.168.203.143@tcp:250:kworker.0 02000000:00000001:2.0:1787828523.879076:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787828523.879087:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787828523.879092:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787828523.879094:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828523.879098:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1787828523.879100:0:40701:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 02000000:00000001:0.0:1787828523.879101:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000008:00000001:2.0:1787828523.879104:0:40701:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828523.879106:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828523.879111:0:40701:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828523.879113:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1787828523.879120:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787828523.879123:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828523.879125:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787828523.879126:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787828523.879130:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828523.879130:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828523.879136:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787828523.879143:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2e940 x1874670646738560/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787828523.879151:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787828523.879153:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828523.879157:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787828523.879159:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1787828523.879162:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00080000:2.0:1787828523.879162:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1787828523.879168:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:0.0:1787828523.879171:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417869248 : -126234291682368 : ffff8d30c9e2c3c0) 00000100:00080000:2.0:1787828523.879188:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787828523.879195:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:2.0:1787828523.879196:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787828523.879205:0:40701:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3324 00000100:00080000:2.0:1787828523.879210:0:40701:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787828523.879213:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828523.879215:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787828523.879218:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787828523.879220:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787828523.879222:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000020:00000001:2.0:1787828523.879225:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:2.0:1787828523.879229:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828523.879231:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:2.0:1787828523.879235:0:40701:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787828523.879238:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828523.879240:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787828523.879245:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828523.879248:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828523.879250:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1787828523.879252:0:40701:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000008:00000001:2.0:1787828523.879256:0:40701:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828523.879258:0:40701:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828523.879263:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787828523.879265:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828523.879267:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787828523.879269:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787828523.879271:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828523.879279:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:0.0:1787828523.879290:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00100000:2.0:1787828523.879292:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2c780 x1874670646738688/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787828523.879305:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787828523.879308:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828523.879315:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1787828523.879317:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828523.879332:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828523.879338:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828523.879342:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787828523.879351:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828523.879358:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4297998080 00000100:00000001:0.0:1787828523.879374:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1787828523.879386:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4297998080 00000100:00000001:0.0:1787828523.879393:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1787828523.879397:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787828523.879401:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828523.879404:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787828523.879434:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828523/real 1787828523] req@ffff8d30c9e2c3c0 x1874670646738304/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828548 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787828523.879450:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787828523.879453:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787828523.879476:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787828523.879480:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787828523.879482:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828523.879486:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828523.879489:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787828523.879497:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2c3c0 x1874670646738304/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828548 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787828523.879514:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828523.879520:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787828523.879525:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828523.879529:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828523.879532:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828523.879537:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787828523.879542:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787828523.879548:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828523.879557:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787828523.879563:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787828523.879572:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646738304:192.168.203.143@tcp:250:kworker.0 00000100:00000001:0.0:1787828523.879582:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1787828523.879585:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787828523.879588:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828523.879594:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787828523.879597:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787828523.879600:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828523.879605:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646738432:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787828523.879613:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787828523.879676:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787828523.879681:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828523.879685:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1787828523.879691:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828523.879705:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787828523.879708:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417880768 : -126234291670848 : ffff8d30c9e2f0c0) 00000100:00000001:0.0:1787828523.879717:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1787828523.879743:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:0.0:1787828523.879753:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1787828523.879758:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828523.879761:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828523.879765:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787828523.879771:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828523.879781:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4297996823 00000100:00000001:0.0:1787828523.879787:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4297998080 00000100:00000001:0.0:1787828523.879790:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4297996823 00000100:00000001:0.0:1787828523.879793:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4297998080 00000100:00000001:0.0:1787828523.879796:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787828523.879799:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828523.879802:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787828523.879808:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828523/real 1787828523] req@ffff8d30c9e2f0c0 x1874670646738432/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828548 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787828523.879819:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787828523.879821:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787828523.879825:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787828523.879827:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787828523.879829:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828523.879832:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828523.879833:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787828523.879838:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2f0c0 x1874670646738432/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828548 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787828523.879856:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828523.879859:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787828523.879862:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828523.879864:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828523.879866:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828523.879868:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787828523.879871:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787828523.879874:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828523.879894:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787828523.879896:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787828523.879900:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646738432:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787828523.879905:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828523.879907:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828523.879910:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787828523.879913:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787828523.879917:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787828523.879920:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787828523.879922:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787828523.879928:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787828523.879930:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787828523.879934:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787828523.879936:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828523.879938:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828523.879939:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787828523.879940:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787828523.879942:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787828523.879943:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787828523.879944:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787828523.879946:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787828523.879947:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787828523.879950:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787828523.879952:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828523.879954:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828523.879964:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787828523.879967:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828523.879969:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787828523.879971:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787828523.879973:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1787828523.879975:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787828523.879976:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828523.879979:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787828523.879981:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787828523.879983:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828523.879986:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646738560:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787828523.879990:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787828523.879992:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787828523.879994:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828523.879996:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1787828523.879999:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828523.880004:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787828523.880006:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417878848 : -126234291672768 : ffff8d30c9e2e940) 00000100:00000001:0.0:1787828523.880010:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1787828523.880024:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:0.0:1787828523.880029:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1787828523.880031:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828523.880033:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828523.880034:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787828523.880037:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828523.880041:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4297996823 00000100:00000001:0.0:1787828523.880044:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4297998080 00000100:00000001:0.0:1787828523.880046:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4297996823 00000100:00000001:0.0:1787828523.880048:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4297998080 00000100:00000001:0.0:1787828523.880051:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787828523.880053:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828523.880055:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787828523.880059:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828523/real 1787828523] req@ffff8d30c9e2e940 x1874670646738560/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828548 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787828523.880073:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787828523.880074:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787828523.880077:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787828523.880079:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787828523.880080:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828523.880082:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828523.880083:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787828523.880087:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2e940 x1874670646738560/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828548 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787828523.880094:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828523.880097:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787828523.880099:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828523.880100:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828523.880102:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828523.880103:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787828523.880105:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787828523.880107:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828523.880110:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787828523.880112:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787828523.880115:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646738560:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787828523.880118:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1787828523.880120:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787828523.880121:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828523.880123:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787828523.880125:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787828523.880128:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828523.880132:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646738688:192.168.203.143@tcp:8:kworker.0 00000100:00000001:0.0:1787828523.880137:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787828523.880140:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787828523.880143:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828523.880146:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1787828523.880150:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828523.880159:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787828523.880161:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417870208 : -126234291681408 : ffff8d30c9e2c780) 00000100:00000001:0.0:1787828523.880167:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1787828523.880185:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:0.0:1787828523.880192:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1787828523.880196:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828523.880198:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828523.880201:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787828523.880205:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828523.880211:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4297996836 00000100:00000001:0.0:1787828523.880215:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4297998080 00000100:00000001:0.0:1787828523.880217:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4297996836 00000100:00000001:0.0:1787828523.880220:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4297998080 00000100:00000001:0.0:1787828523.880223:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787828523.880225:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828523.880226:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787828523.880231:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828523/real 1787828523] req@ffff8d30c9e2c780 x1874670646738688/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828548 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787828523.880240:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787828523.880241:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787828523.880244:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787828523.880246:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787828523.880247:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828523.880249:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828523.880251:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787828523.880255:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2c780 x1874670646738688/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828548 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787828523.880261:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828523.880263:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787828523.880266:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828523.880268:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828523.880270:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828523.880271:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787828523.880274:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787828523.880276:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828523.880279:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787828523.880281:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787828523.880284:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646738688:192.168.203.143@tcp:8:kworker.0 00000100:00000001:0.0:1787828523.880288:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828523.880289:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828523.880291:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787828523.880292:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787828523.880295:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787828523.880296:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787828523.880298:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787828523.880300:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787828523.880302:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787828523.880307:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787828523.880308:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828523.880310:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828523.880311:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787828523.880312:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787828523.880314:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787828523.880315:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787828523.880316:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787828523.880318:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787828523.880319:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787828523.880320:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787828523.880322:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828523.880323:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828523.880326:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787828523.880328:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828523.880330:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787828523.880331:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828524.774731:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828524.774739:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787828524.774857:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787828524.774863:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787828524.774869:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828525.798730:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828525.798738:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787828525.798836:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787828525.798842:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787828525.798849:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828526.822714:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828526.822720:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828526.822797:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828526.822801:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828526.822804:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828527.846729:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828527.846739:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787828527.846845:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787828527.846850:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787828527.846856:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828528.870725:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828528.870733:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787828528.870870:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787828528.870876:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787828528.870881:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787828528.998750:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828528.998765:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1787828528.998768:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787828528.998773:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787828528.998778:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787828528.998786:0:40701:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 3329 00000100:00080000:0.0:1787828528.998800:0:40701:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787828528.998803:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787828528.998805:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787828528.998810:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787828528.998813:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787828528.998816:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:0.0:1787828528.998818:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:0.0:1787828528.998820:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787828528.998822:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:0.0:1787828528.998826:0:40701:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787828528.998829:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828528.998831:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787828528.998835:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787828528.998838:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828528.998840:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1787828528.998842:0:40701:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828528.998858:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787828528.998860:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828528.998864:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787828528.998866:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787828528.998872:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828528.998888:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828528.998899:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d25a40 x1874670646738816/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787828528.998910:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787828528.998915:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828528.998934:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787828528.998938:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828528.998943:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1787828528.998945:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787828528.998947:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787828528.998951:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787828528.998954:0:40701:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3329 00000100:00080000:0.0:1787828528.998959:0:40701:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787828528.998962:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787828528.998964:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787828528.998967:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:2.0:1787828528.998968:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:0.0:1787828528.998969:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787828528.998972:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000020:00000001:0.0:1787828528.998974:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:2.0:1787828528.998975:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787828528.998977:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787828528.998978:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:2.0:1787828528.998979:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1787828528.998983:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787828528.998986:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787828528.998992:0:40701:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 02000000:00000001:2.0:1787828528.998992:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:0.0:1787828528.998995:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828528.998996:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787828528.998997:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787828528.999000:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828528.999004:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00100000:2.0:1787828528.999005:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d25a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646738816:192.168.203.143@tcp:250:kworker.0 02000000:00000001:0.0:1787828528.999007:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828528.999010:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:2.0:1787828528.999012:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000008:00100000:0.0:1787828528.999017:0:40701:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 02000000:00000001:2.0:1787828528.999017:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000008:00000001:0.0:1787828528.999021:0:40701:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828528.999021:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828528.999023:0:40701:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828528.999025:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1787828528.999029:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787828528.999030:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828528.999031:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828528.999033:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787828528.999034:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787828528.999036:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828528.999042:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828528.999048:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d24f00 x1874670646738944/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787828528.999055:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787828528.999057:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601343040 : -126234108208576 : ffff8d30d4d25a40) 00000100:00000001:2.0:1787828528.999062:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:0.0:1787828528.999065:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787828528.999067:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828528.999071:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787828528.999074:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828528.999077:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1787828528.999079:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787828528.999081:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787828528.999084:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787828528.999087:0:40701:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3329 00000100:00080000:0.0:1787828528.999090:0:40701:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787828528.999092:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787828528.999094:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787828528.999097:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787828528.999098:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787828528.999100:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000020:00000001:0.0:1787828528.999103:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:0.0:1787828528.999106:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787828528.999108:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:0.0:1787828528.999111:0:40701:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787828528.999114:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828528.999116:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000800:00000100:2.0:1787828528.999128:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 02000000:00000001:0.0:1787828528.999130:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787828528.999132:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828528.999134:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1787828528.999136:0:40701:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1787828528.999139:0:40701:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828528.999141:0:40701:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828528.999145:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787828528.999146:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828528.999148:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1787828528.999149:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787828528.999151:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787828528.999153:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828528.999153:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828528.999157:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828528.999158:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828528.999159:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00100000:0.0:1787828528.999162:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d24000 x1874670646739072/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787828528.999168:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828528.999172:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787828528.999174:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828528.999177:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787828528.999180:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828528.999184:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1787828528.999184:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4298003200 00000100:00000001:0.0:1787828528.999186:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787828528.999188:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787828528.999191:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787828528.999194:0:40701:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3329 00000100:00080000:0.0:1787828528.999197:0:40701:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787828528.999198:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1787828528.999200:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787828528.999201:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787828528.999204:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787828528.999205:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:2.0:1787828528.999207:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4298003200 00000020:00000001:0.0:1787828528.999208:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000020:00000001:0.0:1787828528.999227:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:2.0:1787828528.999229:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1787828528.999231:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787828528.999233:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:2.0:1787828528.999235:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00080000:0.0:1787828528.999237:0:40701:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787828528.999239:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828528.999241:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828528.999243:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:2.0:1787828528.999244:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 02000000:00000001:0.0:1787828528.999247:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787828528.999250:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787828528.999251:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828528/real 1787828528] req@ffff8d30d4d25a40 x1874670646738816/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828553 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787828528.999253:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1787828528.999257:0:40701:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1787828528.999261:0:40701:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828528.999263:0:40701:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:2.0:1787828528.999267:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 02000000:00000001:0.0:1787828528.999268:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787828528.999270:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:2.0:1787828528.999270:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787828528.999273:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787828528.999275:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:2.0:1787828528.999275:0:2390:0:(events.c:78:reply_in_callback()) Process entered 02000000:00000001:0.0:1787828528.999278:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828528.999279:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787828528.999282:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828528.999284:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828528.999285:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828528.999289:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787828528.999290:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d270c0 x1874670646739200/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787828528.999296:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d25a40 x1874670646738816/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828553 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787828528.999301:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787828528.999304:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828528.999307:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828528.999308:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828528.999331:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787828528.999336:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828528.999339:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828528.999342:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828528.999347:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787828528.999351:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787828528.999357:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828528.999365:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:2.0:1787828528.999370:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1787828528.999429:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d25a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646738816:192.168.203.143@tcp:250:kworker.0 00000100:00000001:2.0:1787828528.999441:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828528.999444:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828528.999446:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787828528.999451:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1787828528.999456:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787828528.999460:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:2.0:1787828528.999463:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:2.0:1787828528.999470:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787828528.999472:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1787828528.999476:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787828528.999478:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828528.999482:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828528.999494:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1787828528.999496:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828528.999499:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787828528.999501:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1787828528.999503:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1787828528.999505:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787828528.999517:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828528.999520:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787828528.999523:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828528.999526:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787828528.999547:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d24f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646738944:192.168.203.143@tcp:38:kworker.0 00000100:00000001:2.0:1787828528.999552:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787828528.999555:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1787828528.999557:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828528.999564:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1787828528.999568:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828528.999577:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787828528.999580:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601340160 : -126234108211456 : ffff8d30d4d24f00) 00000100:00000001:2.0:1787828528.999586:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1787828528.999656:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:2.0:1787828528.999680:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1787828528.999682:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828528.999683:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828528.999685:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1787828528.999688:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828528.999693:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298001943 00000100:00000001:2.0:1787828528.999696:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298003200 00000100:00000001:2.0:1787828528.999698:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298001943 00000100:00000001:2.0:1787828528.999700:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298003200 00000100:00000001:2.0:1787828528.999702:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787828528.999703:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828528.999705:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1787828528.999708:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828528/real 1787828528] req@ffff8d30d4d24f00 x1874670646738944/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828553 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1787828528.999714:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787828528.999715:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1787828528.999717:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1787828528.999719:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787828528.999720:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828528.999722:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828528.999723:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787828528.999726:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d24f00 x1874670646738944/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828553 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787828528.999731:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828528.999733:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787828528.999735:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828528.999736:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828528.999737:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828528.999738:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787828528.999740:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787828528.999743:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828528.999745:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:2.0:1787828528.999747:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1787828528.999750:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d24f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646738944:192.168.203.143@tcp:38:kworker.0 00000100:00000001:2.0:1787828528.999752:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1787828528.999753:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787828528.999754:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828528.999756:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787828528.999757:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828528.999759:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787828528.999761:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d24000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646739072:192.168.203.143@tcp:38:kworker.0 00000100:00000001:2.0:1787828528.999764:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787828528.999765:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1787828528.999767:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828528.999768:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1787828528.999771:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828528.999774:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787828528.999775:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601336320 : -126234108215296 : ffff8d30d4d24000) 00000100:00000001:2.0:1787828528.999779:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1787828528.999788:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:2.0:1787828528.999791:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1787828528.999792:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828528.999794:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828528.999795:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1787828528.999797:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828528.999800:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298001943 00000100:00000001:2.0:1787828528.999802:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298003200 00000100:00000001:2.0:1787828528.999804:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298001943 00000100:00000001:2.0:1787828528.999806:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298003200 00000100:00000001:2.0:1787828528.999808:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787828528.999810:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828528.999812:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1787828528.999814:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828528/real 1787828528] req@ffff8d30d4d24000 x1874670646739072/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828553 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1787828528.999821:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787828528.999822:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1787828528.999824:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1787828528.999825:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787828528.999826:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828528.999827:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828528.999828:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787828528.999831:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d24000 x1874670646739072/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828553 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787828528.999837:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828528.999839:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787828528.999840:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828528.999841:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828528.999842:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828528.999843:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787828528.999844:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787828528.999846:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828528.999848:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:2.0:1787828528.999849:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1787828528.999852:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d24000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646739072:192.168.203.143@tcp:38:kworker.0 00000100:00000001:2.0:1787828528.999854:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1787828528.999855:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787828528.999856:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828528.999858:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787828528.999859:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828528.999860:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787828528.999862:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d270c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646739200:192.168.203.143@tcp:8:kworker.0 00000100:00000001:2.0:1787828528.999865:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787828528.999866:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1787828528.999868:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828528.999869:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1787828528.999871:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828528.999875:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787828528.999876:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601348800 : -126234108202816 : ffff8d30d4d270c0) 00000100:00000001:2.0:1787828528.999879:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1787828528.999885:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:2.0:1787828528.999888:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1787828528.999890:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828528.999891:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828528.999892:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1787828528.999894:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828528.999897:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298001956 00000100:00000001:2.0:1787828528.999899:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298003200 00000100:00000001:2.0:1787828528.999900:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298001956 00000100:00000001:2.0:1787828528.999902:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298003200 00000100:00000001:2.0:1787828528.999903:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787828528.999904:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828528.999906:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1787828528.999908:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828528/real 1787828528] req@ffff8d30d4d270c0 x1874670646739200/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828553 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1787828528.999915:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787828528.999916:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1787828528.999918:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1787828528.999919:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787828528.999920:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828528.999922:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828528.999923:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787828528.999926:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d270c0 x1874670646739200/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828553 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787828528.999936:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828528.999938:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787828528.999940:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828528.999941:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828528.999942:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828528.999943:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787828528.999945:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787828528.999946:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828528.999948:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:2.0:1787828528.999950:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1787828528.999952:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d270c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646739200:192.168.203.143@tcp:8:kworker.0 00000100:00000001:2.0:1787828528.999954:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828528.999956:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828528.999957:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787828528.999958:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1787828528.999959:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787828528.999960:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:2.0:1787828528.999961:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:2.0:1787828528.999963:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787828528.999964:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1787828528.999967:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787828528.999968:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828528.999969:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828528.999970:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787828528.999971:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1787828528.999972:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787828528.999973:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:2.0:1787828528.999974:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:2.0:1787828528.999976:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787828528.999977:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1787828528.999978:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787828528.999979:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828528.999980:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828528.999981:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787828528.999983:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1787828528.999984:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787828528.999985:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:2.0:1787828528.999986:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:2.0:1787828528.999987:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787828528.999988:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1787828528.999992:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787828528.999993:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828528.999994:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828528.999997:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1787828528.999998:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828528.999999:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787828529.000000:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828529.894718:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828529.894726:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828529.894815:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828529.894820:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828529.894824:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828530.918743:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828530.918752:0:815:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1787828530.918755:0:815:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1787828530.918760:0:815:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828530.918766:0:815:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1787828530.918769:0:815:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828530.918772:0:815:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828530.918774:0:815:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787828530.918864:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787828530.918870:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787828530.918874:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828531.942716:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828531.942725:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787828531.942864:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787828531.942872:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787828531.942880:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828532.966694:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828532.966701:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828532.966810:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828532.966815:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828532.966818:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828533.990724:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828533.990731:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787828533.990850:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787828533.990855:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787828533.990859:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787828534.118785:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828534.118799:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1787828534.118803:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1787828534.118836:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787828534.118842:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1787828534.118851:0:468:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 3334 00000100:00080000:1.0:1787828534.118863:0:468:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1787828534.118868:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787828534.118871:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:1.0:1787828534.118876:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:1.0:1787828534.118880:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:1.0:1787828534.118883:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:1.0:1787828534.118887:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:1.0:1787828534.118901:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787828534.118902:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:1.0:1787828534.118907:0:468:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:1.0:1787828534.118909:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828534.118929:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1787828534.118933:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787828534.118937:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828534.118940:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:1.0:1787828534.118943:0:468:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828534.118955:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1787828534.118957:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828534.118961:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1787828534.118963:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1787828534.118968:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828534.118980:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787828534.118990:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d13ef480 x1874670646739328/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787828534.119001:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787828534.119005:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828534.119026:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1787828534.119029:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828534.119038:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1787828534.119040:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1787828534.119042:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787828534.119045:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1787828534.119047:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3334 00000100:00080000:1.0:1787828534.119052:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1787828534.119056:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787828534.119059:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:1.0:1787828534.119063:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:1.0:1787828534.119066:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:1.0:1787828534.119069:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000020:00000001:1.0:1787828534.119075:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:2.0:1787828534.119075:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787828534.119099:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828534.119099:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1787828534.119105:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:2.0:1787828534.119115:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00080000:1.0:1787828534.119121:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787828534.119122:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787828534.119125:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828534.119130:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828534.119133:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787828534.119138:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787828534.119138:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828534.119144:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828534.119149:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00100000:2.0:1787828534.119150:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d13ef480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646739328:192.168.203.143@tcp:250:kworker.0 02000000:00000001:1.0:1787828534.119159:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828534.119160:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:1.0:1787828534.119167:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 02000000:00000001:2.0:1787828534.119167:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1787828534.119173:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000008:00100000:1.0:1787828534.119175:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 02000000:00000001:2.0:1787828534.119177:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000008:00000001:1.0:1787828534.119182:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828534.119184:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828534.119185:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828534.119194:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1787828534.119197:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828534.119199:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1787828534.119201:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1787828534.119220:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828534.119222:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787828534.119226:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839541351552 : -126234168200064 : ffff8d30d13ef480) 00000100:00000001:1.0:1787828534.119228:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828534.119231:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00100000:1.0:1787828534.119249:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d13ef840 x1874670646739456/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787828534.119259:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787828534.119262:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828534.119266:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1787828534.119269:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828534.119272:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:1.0:1787828534.119274:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1787828534.119277:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787828534.119279:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1787828534.119282:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3334 00000100:00080000:1.0:1787828534.119287:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1787828534.119293:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000800:00000100:2.0:1787828534.119293:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787828534.119298:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:1.0:1787828534.119302:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:1.0:1787828534.119303:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:1.0:1787828534.119306:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000020:00000001:1.0:1787828534.119309:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:1.0:1787828534.119312:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828534.119312:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787828534.119317:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:2.0:1787828534.119320:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00080000:1.0:1787828534.119324:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787828534.119327:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828534.119329:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828534.119331:0:2390:0:(events.c:68:request_out_callback()) Process leaving 02000000:00000001:1.0:1787828534.119334:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1787828534.119360:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787828534.119364:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828534.119365:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828534.119372:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:2.0:1787828534.119377:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4298008320 00000008:00100000:1.0:1787828534.119378:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1787828534.119386:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828534.119388:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828534.119394:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:2.0:1787828534.119395:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 02000000:00000001:1.0:1787828534.119396:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828534.119402:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1787828534.119404:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:2.0:1787828534.119406:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4298008320 02000000:00000001:1.0:1787828534.119407:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828534.119418:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828534.119419:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 00000100:00000001:2.0:1787828534.119425:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00100000:1.0:1787828534.119426:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d13ede00 x1874670646739584/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787828534.119431:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828534.119440:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00000001:1.0:1787828534.119445:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787828534.119449:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787828534.119451:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828534/real 1787828534] req@ffff8d30d13ef480 x1874670646739328/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828559 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787828534.119453:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1787828534.119459:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828534.119469:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1787828534.119473:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1787828534.119476:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787828534.119483:0:468:0:(import.c:485:import_select_connection()) Process entered 00000400:00000001:2.0:1787828534.119484:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00080000:1.0:1787828534.119488:0:468:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3334 00000400:00000001:2.0:1787828534.119490:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00080000:1.0:1787828534.119497:0:468:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787828534.119498:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787828534.119502:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828534.119504:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787828534.119506:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:2.0:1787828534.119509:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:1.0:1787828534.119510:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:2.0:1787828534.119514:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000020:00000001:1.0:1787828534.119515:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:2.0:1787828534.119520:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:1.0:1787828534.119521:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000020:00000001:1.0:1787828534.119529:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:1.0:1787828534.119534:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00100000:2.0:1787828534.119535:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d13ef480 x1874670646739328/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828559 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787828534.119537:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:1.0:1787828534.119548:0:468:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:1.0:1787828534.119557:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828534.119562:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:1.0:1787828534.119580:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1787828534.119586:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787828534.119594:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 02000000:00000001:1.0:1787828534.119595:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828534.119600:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828534.119602:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:2.0:1787828534.119604:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828534.119607:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000008:00100000:1.0:1787828534.119608:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00000001:2.0:1787828534.119610:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787828534.119649:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787828534.119654:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828534.119659:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:2.0:1787828534.119662:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1787828534.119666:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d13ef480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646739328:192.168.203.143@tcp:250:kworker.0 00000100:00000001:2.0:1787828534.119673:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828534.119687:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828534.119689:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787828534.119693:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 00000008:00000001:1.0:1787828534.119695:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828534.119699:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00000001:1.0:1787828534.119700:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1787828534.119707:0:2390:0:(genops.c:1217:class_import_put()) Process entered 02000000:00000001:1.0:1787828534.119708:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1787828534.119729:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1787828534.119729:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 00000100:00000001:1.0:1787828534.119735:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1787828534.119737:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1787828534.119740:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828534.119751:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 00000100:00000001:1.0:1787828534.119755:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828534.119757:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1787828534.119765:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00100000:1.0:1787828534.119766:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d13ef0c0 x1874670646739712/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787828534.119771:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828534.119780:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828534.119785:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787828534.119791:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828534.119795:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787828534.119796:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828534.119799:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828534.119804:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787828534.119834:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1787828534.119836:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1787828534.119837:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787828534.119839:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828534.119842:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787828534.119844:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828534.119857:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787828534.119862:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d13ef840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646739456:192.168.203.143@tcp:38:kworker.0 00000100:00000001:2.0:1787828534.119866:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787828534.119869:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1787828534.119881:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828534.119883:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1787828534.119887:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828534.119893:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787828534.119894:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839541352512 : -126234168199104 : ffff8d30d13ef840) 00000100:00000001:2.0:1787828534.119899:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1787828534.119913:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:2.0:1787828534.119918:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1787828534.119920:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828534.119922:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828534.119923:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1787828534.119926:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828534.119930:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298007063 00000100:00000001:2.0:1787828534.119933:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298008320 00000100:00000001:2.0:1787828534.119935:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298007063 00000100:00000001:2.0:1787828534.119937:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298008320 00000100:00000001:2.0:1787828534.119939:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787828534.119941:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828534.119942:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1787828534.119946:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828534/real 1787828534] req@ffff8d30d13ef840 x1874670646739456/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828559 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1787828534.119953:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787828534.119955:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1787828534.119957:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1787828534.119958:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787828534.119960:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828534.119961:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828534.119963:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787828534.119974:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d13ef840 x1874670646739456/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828559 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787828534.119981:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828534.119983:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787828534.119985:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828534.119987:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828534.119989:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828534.119990:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787828534.119992:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787828534.119995:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828534.119998:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:2.0:1787828534.120000:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1787828534.120002:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d13ef840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646739456:192.168.203.143@tcp:38:kworker.0 00000100:00000001:2.0:1787828534.120005:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1787828534.120007:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787828534.120008:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828534.120010:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787828534.120012:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828534.120013:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787828534.120016:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d13ede00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646739584:192.168.203.143@tcp:38:kworker.0 00000100:00000001:2.0:1787828534.120018:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787828534.120020:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1787828534.120022:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828534.120023:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1787828534.120026:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828534.120029:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787828534.120031:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839541345792 : -126234168205824 : ffff8d30d13ede00) 00000100:00000001:2.0:1787828534.120034:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1787828534.120043:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:2.0:1787828534.120047:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1787828534.120049:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828534.120050:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828534.120051:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1787828534.120054:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828534.120057:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298007063 00000100:00000001:2.0:1787828534.120059:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298008320 00000100:00000001:2.0:1787828534.120061:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298007063 00000100:00000001:2.0:1787828534.120063:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298008320 00000100:00000001:2.0:1787828534.120065:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787828534.120067:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828534.120068:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1787828534.120071:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828534/real 1787828534] req@ffff8d30d13ede00 x1874670646739584/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828559 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1787828534.120078:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787828534.120079:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1787828534.120080:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1787828534.120082:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787828534.120083:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828534.120084:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828534.120085:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787828534.120088:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d13ede00 x1874670646739584/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828559 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787828534.120105:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828534.120106:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787828534.120108:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828534.120110:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828534.120111:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828534.120112:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787828534.120113:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787828534.120115:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828534.120117:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:2.0:1787828534.120119:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1787828534.120121:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d13ede00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646739584:192.168.203.143@tcp:38:kworker.0 00000100:00000001:2.0:1787828534.120124:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1787828534.120125:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787828534.120127:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828534.120128:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787828534.120130:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828534.120132:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787828534.120134:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d13ef0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646739712:192.168.203.143@tcp:8:kworker.0 00000100:00000001:2.0:1787828534.120137:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787828534.120139:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1787828534.120140:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828534.120142:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1787828534.120144:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828534.120147:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787828534.120148:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839541350592 : -126234168201024 : ffff8d30d13ef0c0) 00000100:00000001:2.0:1787828534.120151:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1787828534.120177:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:2.0:1787828534.120181:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1787828534.120183:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828534.120184:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828534.120186:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1787828534.120188:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828534.120191:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298007076 00000100:00000001:2.0:1787828534.120193:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298008320 00000100:00000001:2.0:1787828534.120196:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298007076 00000100:00000001:2.0:1787828534.120198:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298008320 00000100:00000001:2.0:1787828534.120200:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787828534.120202:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828534.120203:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1787828534.120207:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828534/real 1787828534] req@ffff8d30d13ef0c0 x1874670646739712/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828559 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1787828534.120214:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787828534.120215:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1787828534.120216:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1787828534.120218:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787828534.120219:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828534.120220:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828534.120222:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787828534.120225:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d13ef0c0 x1874670646739712/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828559 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787828534.120238:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828534.120240:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787828534.120253:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828534.120255:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828534.120256:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828534.120257:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787828534.120259:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787828534.120261:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828534.120264:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:2.0:1787828534.120266:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1787828534.120268:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d13ef0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646739712:192.168.203.143@tcp:8:kworker.0 00000100:00000001:2.0:1787828534.120272:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828534.120274:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828534.120275:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787828534.120276:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1787828534.120278:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787828534.120280:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:2.0:1787828534.120281:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:2.0:1787828534.120283:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787828534.120284:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1787828534.120286:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787828534.120287:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828534.120288:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828534.120289:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787828534.120290:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1787828534.120292:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787828534.120293:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:2.0:1787828534.120294:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:2.0:1787828534.120296:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787828534.120297:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1787828534.120300:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787828534.120302:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828534.120303:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828534.120304:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787828534.120306:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1787828534.120307:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787828534.120309:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:2.0:1787828534.120310:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:2.0:1787828534.120311:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787828534.120312:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1787828534.120314:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787828534.120315:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828534.120317:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828534.120320:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1787828534.120321:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828534.120323:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787828534.120324:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828535.014668:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828535.014674:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787828535.014766:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787828535.014770:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787828535.014773:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828536.038676:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828536.038683:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828536.038730:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828536.038734:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828536.038736:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828537.062684:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828537.062690:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787828537.062750:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787828537.062753:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787828537.062756:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828538.086706:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828538.086714:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787828538.086771:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787828538.086777:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787828538.086780:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828539.110714:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828539.110725:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828539.110809:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828539.110815:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828539.110820:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787828539.238749:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828539.238758:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1787828539.238761:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787828539.238766:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787828539.238770:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787828539.238779:0:468:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 3339 00000100:00080000:3.0:1787828539.238790:0:468:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787828539.238794:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787828539.238797:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:3.0:1787828539.238801:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1787828539.238803:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1787828539.238806:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:3.0:1787828539.238808:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:3.0:1787828539.238810:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787828539.238811:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:3.0:1787828539.238816:0:468:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:3.0:1787828539.238819:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787828539.238821:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787828539.238824:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787828539.238827:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828539.238829:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:3.0:1787828539.238831:0:468:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787828539.238840:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787828539.238841:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828539.238844:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787828539.238846:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787828539.238850:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828539.238861:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787828539.238872:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c7015680 x1874670646739840/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787828539.238886:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787828539.238891:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828539.238902:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787828539.238906:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828539.238909:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1787828539.238910:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787828539.238912:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787828539.238916:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:2.0:1787828539.238917:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00080000:3.0:1787828539.238921:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3339 00000100:00000001:2.0:1787828539.238923:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1787828539.238926:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00080000:3.0:1787828539.238926:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787828539.238928:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787828539.238931:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828539.238932:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828539.238934:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 02000000:00000001:2.0:1787828539.238937:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000020:00000001:3.0:1787828539.238938:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1787828539.238941:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 02000000:00000001:2.0:1787828539.238942:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000020:00000001:3.0:1787828539.238944:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 02000000:00000001:2.0:1787828539.238946:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828539.238948:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:3.0:1787828539.238952:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00100000:2.0:1787828539.238953:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c7015680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646739840:192.168.203.143@tcp:250:kworker.0 00000100:00000001:3.0:1787828539.238956:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:2.0:1787828539.238960:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00080000:3.0:1787828539.238962:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 02000000:00000001:2.0:1787828539.238966:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:3.0:1787828539.238966:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828539.238970:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787828539.238970:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787828539.238974:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1787828539.238977:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828539.238978:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787828539.238983:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828539.238986:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:2.0:1787828539.238994:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787828539.238997:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369541248 : -126234340010368 : ffff8d30c7015680) 00000100:00000001:2.0:1787828539.239000:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1787828539.239032:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:2.0:1787828539.239041:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1787828539.239044:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828539.239046:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828539.239048:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1787828539.239081:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000008:00100000:3.0:1787828539.239084:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00000001:2.0:1787828539.239085:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4298013440 00000008:00000001:3.0:1787828539.239092:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828539.239096:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 00000100:00000001:3.0:1787828539.239096:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828539.239103:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4298013440 02000000:00000001:3.0:1787828539.239104:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:2.0:1787828539.239108:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 02000000:00000001:3.0:1787828539.239109:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828539.239112:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787828539.239114:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:2.0:1787828539.239127:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787828539.239129:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:2.0:1787828539.239131:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1787828539.239136:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828539/real 1787828539] req@ffff8d30c7015680 x1874670646739840/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828564 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1787828539.239144:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787828539.239146:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1787828539.239148:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1787828539.239150:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787828539.239151:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828539.239153:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828539.239154:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787828539.239158:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c7015680 x1874670646739840/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828564 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787828539.239164:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828539.239166:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787828539.239168:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828539.239170:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828539.239170:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828539.239172:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787828539.239174:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787828539.239177:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828539.239180:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:2.0:1787828539.239181:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1787828539.239184:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c7015680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646739840:192.168.203.143@tcp:250:kworker.0 00000100:00000001:2.0:1787828539.239188:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828539.239190:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828539.239201:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787828539.239203:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1787828539.239205:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787828539.239207:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:2.0:1787828539.239208:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:2.0:1787828539.239211:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787828539.239213:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1787828539.239214:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787828539.239215:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828539.239217:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828539.239223:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1787828539.239225:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828539.239227:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787828539.239228:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787828539.239250:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828539.239256:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787828539.239262:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c70143c0 x1874670646739968/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787828539.239280:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787828539.239282:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828539.239288:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787828539.239290:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828539.239293:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1787828539.239295:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787828539.239296:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787828539.239298:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:2.0:1787828539.239320:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787828539.239323:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1787828539.239325:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1787828539.239326:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787828539.239328:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828539.239330:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787828539.239332:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828539.239334:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787828539.239353:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c70143c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646739968:192.168.203.143@tcp:38:kworker.0 00000100:00000001:2.0:1787828539.239356:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787828539.239359:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1787828539.239360:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828539.239362:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1787828539.239364:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828539.239369:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00080000:3.0:1787828539.239369:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3339 00000100:00000001:2.0:1787828539.239373:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369536448 : -126234340015168 : ffff8d30c70143c0) 00000100:00000001:2.0:1787828539.239377:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00080000:3.0:1787828539.239377:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787828539.239382:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787828539.239384:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:3.0:1787828539.239386:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1787828539.239389:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1787828539.239391:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000020:00000001:3.0:1787828539.239393:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000800:00000100:2.0:1787828539.239395:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:3.0:1787828539.239396:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787828539.239399:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:2.0:1787828539.239403:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00080000:3.0:1787828539.239404:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787828539.239407:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787828539.239408:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828539.239410:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787828539.239411:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:2.0:1787828539.239414:0:2390:0:(events.c:68:request_out_callback()) Process leaving 02000000:00000001:3.0:1787828539.239417:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787828539.239419:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787828539.239422:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828539.239425:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298012182 00000100:00000001:3.0:1787828539.239425:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:3.0:1787828539.239430:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1787828539.239432:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828539.239433:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828539.239437:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298013440 02000000:00000001:3.0:1787828539.239438:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787828539.239440:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828539.239442:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298012182 00000100:00000001:3.0:1787828539.239443:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787828539.239445:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787828539.239453:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828539.239456:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298013440 00000100:00000001:2.0:1787828539.239459:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787828539.239459:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828539.239465:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828539.239468:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1787828539.239469:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c7014f00 x1874670646740096/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787828539.239474:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828539/real 1787828539] req@ffff8d30c70143c0 x1874670646739968/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828564 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787828539.239487:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000400:00000001:2.0:1787828539.239491:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000001:3.0:1787828539.239492:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:2.0:1787828539.239493:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787828539.239497:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828539.239498:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1787828539.239502:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00080000:3.0:1787828539.239503:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:2.0:1787828539.239504:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828539.239508:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00080000:3.0:1787828539.239510:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1787828539.239511:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787828539.239514:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00100000:2.0:1787828539.239517:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c70143c0 x1874670646739968/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828564 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828539.239518:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787828539.239523:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787828539.239529:0:468:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3339 00000100:00000001:2.0:1787828539.239530:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:3.0:1787828539.239533:0:468:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787828539.239534:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787828539.239539:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787828539.239539:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787828539.239542:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:2.0:1787828539.239543:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828539.239545:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:3.0:1787828539.239547:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:2.0:1787828539.239548:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000020:00000001:3.0:1787828539.239551:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:2.0:1787828539.239552:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000020:00000001:3.0:1787828539.239554:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00080000:2.0:1787828539.239556:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000020:00000001:3.0:1787828539.239558:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00080000:2.0:1787828539.239562:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787828539.239562:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787828539.239566:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:2.0:1787828539.239567:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1787828539.239575:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c70143c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646739968:192.168.203.143@tcp:38:kworker.0 00000100:00080000:3.0:1787828539.239575:0:468:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787828539.239581:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787828539.239582:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828539.239585:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 02000000:00000001:3.0:1787828539.239586:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:2.0:1787828539.239588:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787828539.239590:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828539.239591:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1787828539.239595:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 02000000:00000001:3.0:1787828539.239595:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1787828539.239619:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:2.0:1787828539.239621:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:2.0:1787828539.239623:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787828539.239636:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1787828539.239639:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787828539.239639:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:2.0:1787828539.239640:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000008:00100000:3.0:1787828539.239643:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00000001:2.0:1787828539.239644:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000008:00000001:3.0:1787828539.239650:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828539.239651:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787828539.239652:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828539.239655:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787828539.239658:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:2.0:1787828539.239659:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:3.0:1787828539.239661:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828539.239662:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1787828539.239665:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787828539.239665:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787828539.239668:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:2.0:1787828539.239669:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787828539.239673:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787828539.239673:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828539.239680:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787828539.239682:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787828539.239683:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828539.239686:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787828539.239692:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c7015a40 x1874670646740224/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787828539.239693:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c7014f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646740096:192.168.203.143@tcp:38:kworker.0 00000100:00000001:2.0:1787828539.239701:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787828539.239704:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:3.0:1787828539.239708:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 02000000:00000001:2.0:1787828539.239709:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828539.239712:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:3.0:1787828539.239713:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828539.239716:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828539.239718:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828539.239722:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787828539.239725:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369539328 : -126234340012288 : ffff8d30c7014f00) 00000100:00000001:2.0:1787828539.239727:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1787828539.239738:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:2.0:1787828539.239743:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1787828539.239745:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828539.239746:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828539.239748:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1787828539.239750:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828539.239752:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298012183 00000100:00000001:2.0:1787828539.239754:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298013440 00000100:00000001:2.0:1787828539.239756:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298012183 00000100:00000001:2.0:1787828539.239757:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298013440 00000100:00000001:2.0:1787828539.239759:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787828539.239760:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828539.239761:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1787828539.239764:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828539/real 1787828539] req@ffff8d30c7014f00 x1874670646740096/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828564 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1787828539.239769:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787828539.239770:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1787828539.239771:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1787828539.239773:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787828539.239774:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828539.239775:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828539.239776:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787828539.239779:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c7014f00 x1874670646740096/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828564 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787828539.239783:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828539.239785:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787828539.239786:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828539.239787:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828539.239788:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828539.239789:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787828539.239790:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787828539.239792:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828539.239794:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:2.0:1787828539.239795:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1787828539.239798:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c7014f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646740096:192.168.203.143@tcp:38:kworker.0 00000100:00000001:2.0:1787828539.239800:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828539.239801:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828539.239802:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787828539.239803:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1787828539.239804:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787828539.239805:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:2.0:1787828539.239806:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:2.0:1787828539.239807:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787828539.239807:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1787828539.239808:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787828539.239809:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828539.239810:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828539.239812:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1787828539.239813:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828539.239814:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787828539.239815:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1787828539.239816:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1787828539.239817:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787828539.239818:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828539.239820:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787828539.239821:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828539.239822:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787828539.239824:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c7015a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646740224:192.168.203.143@tcp:8:kworker.0 00000100:00000001:2.0:1787828539.239826:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787828539.239827:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1787828539.239828:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828539.239829:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1787828539.239831:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828539.239834:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787828539.239835:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369542208 : -126234340009408 : ffff8d30c7015a40) 00000100:00000001:2.0:1787828539.239837:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1787828539.239843:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:2.0:1787828539.239846:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1787828539.239847:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828539.239848:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828539.239850:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1787828539.239851:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828539.239853:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298012196 00000100:00000001:2.0:1787828539.239855:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298013440 00000100:00000001:2.0:1787828539.239856:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298012196 00000100:00000001:2.0:1787828539.239858:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298013440 00000100:00000001:2.0:1787828539.239859:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787828539.239860:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828539.239862:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1787828539.239864:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828539/real 1787828539] req@ffff8d30c7015a40 x1874670646740224/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828564 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1787828539.239869:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787828539.239869:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1787828539.239871:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1787828539.239872:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787828539.239873:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828539.239874:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828539.239875:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787828539.239877:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c7015a40 x1874670646740224/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828564 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787828539.239881:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828539.239883:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787828539.239884:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828539.239885:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828539.239886:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828539.239887:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787828539.239888:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787828539.239889:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828539.239892:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:2.0:1787828539.239893:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1787828539.239895:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c7015a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646740224:192.168.203.143@tcp:8:kworker.0 00000100:00000001:2.0:1787828539.239897:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828539.239899:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828539.239899:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787828539.239900:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1787828539.239901:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787828539.239902:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:2.0:1787828539.239903:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:2.0:1787828539.239904:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787828539.239905:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1787828539.239905:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787828539.239906:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828539.239907:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828539.239909:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1787828539.239910:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828539.239911:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787828539.239912:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828540.134746:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828540.134754:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787828540.134852:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787828540.134857:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787828540.134862:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828541.158710:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828541.158719:0:815:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1787828541.158721:0:815:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1787828541.158725:0:815:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828541.158730:0:815:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1787828541.158733:0:815:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828541.158735:0:815:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828541.158738:0:815:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787828541.158822:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787828541.158846:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787828541.158852:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828542.182703:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828542.182708:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828542.182772:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828542.182775:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828542.182779:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828543.206743:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828543.206754:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787828543.206858:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787828543.206863:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787828543.206868:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828544.230759:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828544.230769:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787828544.230880:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787828544.230888:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787828544.230895:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787828544.358713:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828544.358724:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1787828544.358727:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1787828544.358731:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787828544.358737:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1787828544.358755:0:468:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 3344 00000100:00080000:1.0:1787828544.358766:0:468:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1787828544.358770:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787828544.358773:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:1.0:1787828544.358778:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:1.0:1787828544.358782:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:1.0:1787828544.358786:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:1.0:1787828544.358800:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:1.0:1787828544.358803:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787828544.358805:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:1.0:1787828544.358820:0:468:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:1.0:1787828544.358823:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828544.358826:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1787828544.358830:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787828544.358833:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828544.358837:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:1.0:1787828544.358839:0:468:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828544.358853:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1787828544.358856:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828544.358861:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1787828544.358865:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1787828544.358871:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828544.358896:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787828544.358910:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d13efc00 x1874670646740352/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787828544.358927:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787828544.358934:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828544.358951:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1787828544.358954:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828544.359006:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1787828544.359009:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1787828544.359011:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787828544.359015:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1787828544.359019:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3344 00000100:00080000:1.0:1787828544.359026:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1787828544.359028:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787828544.359030:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:1.0:1787828544.359033:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:1.0:1787828544.359036:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:1.0:1787828544.359039:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000020:00000001:1.0:1787828544.359042:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:1.0:1787828544.359045:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787828544.359048:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:1.0:1787828544.359054:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:1.0:1787828544.359057:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828544.359059:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1787828544.359068:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787828544.359071:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828544.359074:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:1.0:1787828544.359081:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00000001:2.0:1787828544.359082:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000008:00000001:1.0:1787828544.359089:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828544.359096:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1787828544.359097:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828544.359106:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 02000000:00000001:1.0:1787828544.359109:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:2.0:1787828544.359114:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 02000000:00000001:1.0:1787828544.359115:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828544.359134:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828544.359136:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1787828544.359145:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787828544.359149:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787828544.359155:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828544.359158:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828544.359169:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828544.359173:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787828544.359185:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d13efc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646740352:192.168.203.143@tcp:250:kworker.0 00000100:00100000:1.0:1787828544.359197:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d13ed2c0 x1874670646740480/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787828544.359211:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787828544.359220:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1787828544.359227:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828544.359232:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 02000000:00000001:2.0:1787828544.359237:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:1.0:1787828544.359239:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828544.359248:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828544.359251:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1787828544.359253:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828544.359263:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:1.0:1787828544.359309:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1787828544.359324:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787828544.359328:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1787828544.359331:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3344 00000100:00000001:2.0:1787828544.359335:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00080000:1.0:1787828544.359337:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787828544.359341:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839541353472 : -126234168198144 : ffff8d30d13efc00) 00000100:00000001:1.0:1787828544.359343:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787828544.359347:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:1.0:1787828544.359352:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:2.0:1787828544.359354:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000020:00000001:1.0:1787828544.359355:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:1.0:1787828544.359361:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000020:00000001:1.0:1787828544.359364:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:1.0:1787828544.359366:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787828544.359368:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:1.0:1787828544.359372:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:1.0:1787828544.359375:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828544.359377:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1787828544.359389:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787828544.359392:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828544.359394:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:1.0:1787828544.359406:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1787828544.359411:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828544.359414:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828544.359422:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000800:00000100:2.0:1787828544.359426:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 02000000:00000001:1.0:1787828544.359427:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828544.359435:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1787828544.359437:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1787828544.359442:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828544.359449:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787828544.359452:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828544.359458:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828544.359463:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787828544.359467:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d13ed680 x1874670646740608/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787828544.359469:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1787828544.359480:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828544.359490:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787828544.359498:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828544.359500:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4298018560 00000100:00000001:1.0:1787828544.359503:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1787828544.359511:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828544.359515:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1787828544.359517:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1787828544.359520:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787828544.359520:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 00000100:00000001:1.0:1787828544.359529:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1787828544.359533:0:468:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3344 00000100:00080000:1.0:1787828544.359538:0:468:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1787828544.359541:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787828544.359543:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:2.0:1787828544.359544:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4298018560 00000020:00000001:1.0:1787828544.359546:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:1.0:1787828544.359551:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:1.0:1787828544.359555:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:2.0:1787828544.359557:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 00000020:00000001:1.0:1787828544.359559:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:1.0:1787828544.359565:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787828544.359569:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:2.0:1787828544.359569:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787828544.359579:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1787828544.359580:0:468:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:1.0:1787828544.359589:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828544.359589:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 02000000:00000001:1.0:1787828544.359596:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1787828544.359628:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787828544.359631:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828544.359634:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:1.0:1787828544.359637:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000008:00000001:1.0:1787828544.359639:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828544.359642:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828544.359648:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1787828544.359652:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828544.359657:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00100000:2.0:1787828544.359657:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828544/real 1787828544] req@ffff8d30d13efc00 x1874670646740352/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828569 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:1.0:1787828544.359660:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1787828544.359665:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828544.359674:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787828544.359687:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d13ec3c0 x1874670646740736/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000001:2.0:1787828544.359687:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787828544.359693:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787828544.359704:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787828544.359704:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787828544.359710:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828544.359713:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787828544.359715:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828544.359722:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828544.359729:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828544.359733:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787828544.359750:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d13efc00 x1874670646740352/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828569 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787828544.359762:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828544.359767:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787828544.359771:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828544.359774:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828544.359776:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828544.359780:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787828544.359782:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787828544.359787:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828544.359794:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:2.0:1787828544.359798:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1787828544.359803:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d13efc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646740352:192.168.203.143@tcp:250:kworker.0 00000100:00000001:2.0:1787828544.359811:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828544.359814:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828544.359816:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787828544.359820:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1787828544.359825:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787828544.359828:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:2.0:1787828544.359830:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:2.0:1787828544.359836:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787828544.359839:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1787828544.359843:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787828544.359845:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828544.359849:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828544.359857:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1787828544.359861:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828544.359864:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787828544.359867:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1787828544.359869:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1787828544.359871:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787828544.359874:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828544.359877:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787828544.359880:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828544.359884:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787828544.359889:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d13ed2c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646740480:192.168.203.143@tcp:38:kworker.0 00000100:00000001:2.0:1787828544.359894:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787828544.359897:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1787828544.359900:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828544.359903:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1787828544.359907:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828544.359916:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787828544.359918:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839541342912 : -126234168208704 : ffff8d30d13ed2c0) 00000100:00000001:2.0:1787828544.359924:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1787828544.359945:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:2.0:1787828544.359952:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1787828544.359956:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828544.359958:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828544.359961:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1787828544.359966:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828544.359980:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298017303 00000100:00000001:2.0:1787828544.359985:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298018560 00000100:00000001:2.0:1787828544.359988:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298017303 00000100:00000001:2.0:1787828544.359992:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298018560 00000100:00000001:2.0:1787828544.359995:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787828544.359998:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828544.360001:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1787828544.360007:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828544/real 1787828544] req@ffff8d30d13ed2c0 x1874670646740480/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828569 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1787828544.360020:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787828544.360022:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1787828544.360026:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1787828544.360029:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787828544.360031:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828544.360034:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828544.360037:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787828544.360042:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d13ed2c0 x1874670646740480/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828569 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787828544.360070:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828544.360074:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787828544.360078:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828544.360082:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828544.360084:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828544.360088:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787828544.360091:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787828544.360096:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828544.360102:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:2.0:1787828544.360106:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1787828544.360112:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d13ed2c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646740480:192.168.203.143@tcp:38:kworker.0 00000100:00000001:2.0:1787828544.360119:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1787828544.360133:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787828544.360136:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828544.360140:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787828544.360144:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828544.360148:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787828544.360154:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d13ed680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646740608:192.168.203.143@tcp:38:kworker.0 00000100:00000001:2.0:1787828544.360160:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787828544.360163:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1787828544.360166:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828544.360170:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1787828544.360175:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828544.360184:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787828544.360187:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839541343872 : -126234168207744 : ffff8d30d13ed680) 00000100:00000001:2.0:1787828544.360193:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1787828544.360215:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:2.0:1787828544.360224:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1787828544.360238:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828544.360241:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828544.360244:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1787828544.360248:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828544.360255:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298017303 00000100:00000001:2.0:1787828544.360260:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298018560 00000100:00000001:2.0:1787828544.360264:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298017303 00000100:00000001:2.0:1787828544.360299:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298018560 00000100:00000001:2.0:1787828544.360304:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787828544.360308:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828544.360329:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1787828544.360336:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828544/real 1787828544] req@ffff8d30d13ed680 x1874670646740608/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828569 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1787828544.360348:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787828544.360350:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1787828544.360355:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1787828544.360357:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787828544.360360:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828544.360362:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828544.360364:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787828544.360370:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d13ed680 x1874670646740608/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828569 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787828544.360382:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828544.360385:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787828544.360389:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828544.360391:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828544.360393:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828544.360396:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787828544.360398:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787828544.360402:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828544.360407:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:2.0:1787828544.360411:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1787828544.360416:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d13ed680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646740608:192.168.203.143@tcp:38:kworker.0 00000100:00000001:2.0:1787828544.360422:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1787828544.360425:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787828544.360427:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828544.360430:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787828544.360433:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828544.360436:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787828544.360441:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d13ec3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646740736:192.168.203.143@tcp:8:kworker.0 00000100:00000001:2.0:1787828544.360446:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787828544.360449:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1787828544.360452:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828544.360454:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1787828544.360459:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828544.360467:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787828544.360469:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839541339072 : -126234168212544 : ffff8d30d13ec3c0) 00000100:00000001:2.0:1787828544.360473:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1787828544.360491:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:2.0:1787828544.360499:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1787828544.360502:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828544.360505:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828544.360507:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1787828544.360512:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828544.360517:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298017316 00000100:00000001:2.0:1787828544.360522:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298018560 00000100:00000001:2.0:1787828544.360525:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298017316 00000100:00000001:2.0:1787828544.360529:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298018560 00000100:00000001:2.0:1787828544.360532:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787828544.360535:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828544.360538:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1787828544.360545:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828544/real 1787828544] req@ffff8d30d13ec3c0 x1874670646740736/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828569 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1787828544.360570:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787828544.360573:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1787828544.360577:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1787828544.360580:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787828544.360583:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828544.360585:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828544.360588:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787828544.360594:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d13ec3c0 x1874670646740736/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828569 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787828544.360605:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828544.360609:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787828544.360613:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828544.360646:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828544.360649:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828544.360653:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787828544.360655:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787828544.360660:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828544.360666:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:2.0:1787828544.360670:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1787828544.360675:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d13ec3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646740736:192.168.203.143@tcp:8:kworker.0 00000100:00000001:2.0:1787828544.360682:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828544.360685:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828544.360687:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787828544.360689:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1787828544.360694:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787828544.360696:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:2.0:1787828544.360699:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:2.0:1787828544.360702:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787828544.360705:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1787828544.360709:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787828544.360711:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828544.360714:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828544.360716:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787828544.360718:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1787828544.360721:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787828544.360723:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:2.0:1787828544.360736:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:2.0:1787828544.360740:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787828544.360742:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1787828544.360744:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787828544.360746:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828544.360749:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828544.360751:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787828544.360753:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1787828544.360756:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787828544.360758:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:2.0:1787828544.360760:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:2.0:1787828544.360762:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787828544.360764:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1787828544.360767:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787828544.360769:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828544.360772:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828544.360778:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1787828544.360780:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828544.360783:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787828544.360786:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828545.254777:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828545.254788:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828545.254912:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828545.254918:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828545.254924:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828546.278722:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828546.278730:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787828546.278881:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787828546.278888:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787828546.278892:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828547.302731:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828547.302738:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787828547.302857:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787828547.302868:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787828547.302893:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828548.326708:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828548.326716:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828548.326859:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828548.326867:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828548.326874:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828549.350712:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828549.350718:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787828549.350844:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787828549.350853:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787828549.350860:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787828549.478814:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828549.478828:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1787828549.478832:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787828549.478837:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787828549.478844:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787828549.478872:0:468:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 3349 00000100:00080000:3.0:1787828549.478890:0:468:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787828549.478895:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787828549.478898:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:3.0:1787828549.478904:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1787828549.478908:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1787828549.478912:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:3.0:1787828549.478916:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:3.0:1787828549.478919:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787828549.478920:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:3.0:1787828549.478927:0:468:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:3.0:1787828549.478930:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787828549.478934:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787828549.478939:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787828549.478943:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828549.478946:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:3.0:1787828549.478950:0:468:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787828549.478968:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787828549.478970:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828549.478975:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787828549.478979:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787828549.478985:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828549.479003:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787828549.479017:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c70161c0 x1874670646740864/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787828549.479033:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787828549.479040:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828549.479081:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787828549.479085:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828549.479089:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1787828549.479091:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787828549.479093:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787828549.479096:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787828549.479099:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3349 00000100:00080000:3.0:1787828549.479103:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787828549.479105:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787828549.479108:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:3.0:1787828549.479128:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1787828549.479132:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1787828549.479135:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000020:00000001:3.0:1787828549.479139:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:3.0:1787828549.479142:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787828549.479143:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:3.0:1787828549.479148:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787828549.479151:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1787828549.479152:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787828549.479164:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:2.0:1787828549.479166:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1787828549.479172:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1787828549.479175:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 02000000:00000001:3.0:1787828549.479175:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787828549.479188:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787828549.479189:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828549.479210:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 02000000:00000001:2.0:1787828549.479214:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787828549.479221:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000008:00100000:3.0:1787828549.479221:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 02000000:00000001:2.0:1787828549.479249:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787828549.479291:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c70161c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646740864:192.168.203.143@tcp:250:kworker.0 00000008:00000001:3.0:1787828549.479295:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828549.479311:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828549.479313:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787828549.479337:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1787828549.479342:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787828549.479345:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787828549.479347:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828549.479357:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:3.0:1787828549.479358:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787828549.479374:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787828549.479379:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828549.479383:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828549.479392:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787828549.479406:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c7016580 x1874670646740992/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787828549.479427:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787828549.479431:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369544128 : -126234340007488 : ffff8d30c70161c0) 00000100:00000001:3.0:1787828549.479433:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787828549.479446:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828549.479454:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828549.479455:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00080000:3.0:1787828549.479459:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828549.479465:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1787828549.479467:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787828549.479470:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787828549.479477:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787828549.479482:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3349 00000100:00080000:3.0:1787828549.479490:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787828549.479493:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787828549.479496:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:3.0:1787828549.479500:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1787828549.479507:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000800:00000100:2.0:1787828549.479508:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000020:00000001:3.0:1787828549.479514:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000020:00000001:3.0:1787828549.479520:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:3.0:1787828549.479523:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787828549.479526:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:2.0:1787828549.479527:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00080000:3.0:1787828549.479539:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787828549.479540:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828549.479548:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828549.479548:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787828549.479570:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:2.0:1787828549.479571:0:2390:0:(events.c:68:request_out_callback()) Process leaving 02000000:00000001:3.0:1787828549.479588:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787828549.479589:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787828549.479603:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828549.479606:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:3.0:1787828549.479612:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00000001:2.0:1787828549.479651:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4298023681 00000100:00000001:2.0:1787828549.479665:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 00000100:00000001:2.0:1787828549.479692:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4298023681 00000008:00000001:3.0:1787828549.479694:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828549.479703:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 00000100:00000001:3.0:1787828549.479705:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828549.479725:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787828549.479729:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:2.0:1787828549.479731:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787828549.479733:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828549.479736:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00000001:3.0:1787828549.479739:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787828549.479742:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787828549.479745:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828549.479755:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787828549.479757:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828549/real 1787828549] req@ffff8d30c70161c0 x1874670646740864/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828574 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828549.479769:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c7015a40 x1874670646741120/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000001:2.0:1787828549.479810:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787828549.479815:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787828549.479815:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787828549.479823:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828549.479826:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787828549.479829:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828549.479832:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00080000:3.0:1787828549.479836:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:2.0:1787828549.479838:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828549.479859:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828549.479861:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:3.0:1787828549.479868:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1787828549.479876:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00100000:2.0:1787828549.479878:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c70161c0 x1874670646740864/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828574 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828549.479883:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787828549.479899:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787828549.479914:0:468:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3349 00000100:00000001:2.0:1787828549.479930:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:3.0:1787828549.479931:0:468:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787828549.479938:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787828549.479939:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787828549.479943:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:2.0:1787828549.479949:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:3.0:1787828549.479950:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1787828549.479963:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:2.0:1787828549.479964:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828549.479979:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:3.0:1787828549.479980:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000020:00000001:3.0:1787828549.479997:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:2.0:1787828549.479998:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787828549.480008:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828549.480009:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787828549.480015:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:2.0:1787828549.480020:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828549.480026:0:468:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:3.0:1787828549.480030:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787828549.480036:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00080000:2.0:1787828549.480041:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:2.0:1787828549.480061:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 02000000:00000001:3.0:1787828549.480063:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787828549.480074:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828549.480078:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00100000:2.0:1787828549.480081:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c70161c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646740864:192.168.203.143@tcp:250:kworker.0 00000008:00100000:3.0:1787828549.480082:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1787828549.480091:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828549.480100:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828549.480112:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:3.0:1787828549.480112:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787828549.480118:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828549.480125:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:2.0:1787828549.480127:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828549.480143:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787828549.480146:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787828549.480150:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828549.480151:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 00000100:00000001:3.0:1787828549.480159:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828549.480162:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00100000:3.0:1787828549.480171:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c7014f00 x1874670646741248/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000020:00000001:2.0:1787828549.480172:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:2.0:1787828549.480182:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 00000100:00000001:3.0:1787828549.480197:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 02000000:00000001:2.0:1787828549.480198:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 00000100:00000001:3.0:1787828549.480206:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828549.480208:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787828549.480212:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828549.480217:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787828549.480223:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828549.480227:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828549.480256:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1787828549.480260:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828549.480284:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787828549.480290:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1787828549.480292:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1787828549.480296:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787828549.480299:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828549.480304:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787828549.480308:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828549.480312:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787828549.480322:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c7016580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646740992:192.168.203.143@tcp:38:kworker.0 00000100:00000001:2.0:1787828549.480330:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787828549.480334:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1787828549.480337:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828549.480341:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1787828549.480348:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828549.480359:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787828549.480362:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369545088 : -126234340006528 : ffff8d30c7016580) 00000100:00000001:2.0:1787828549.480370:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1787828549.480400:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:2.0:1787828549.480425:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1787828549.480429:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828549.480430:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828549.480432:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1787828549.480435:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828549.480440:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298022423 00000100:00000001:2.0:1787828549.480444:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298023681 00000100:00000001:2.0:1787828549.480446:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298022423 00000100:00000001:2.0:1787828549.480448:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298023681 00000100:00000001:2.0:1787828549.480451:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787828549.480452:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828549.480454:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1787828549.480459:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828549/real 1787828549] req@ffff8d30c7016580 x1874670646740992/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828574 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1787828549.480468:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787828549.480469:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1787828549.480472:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1787828549.480473:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787828549.480475:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828549.480476:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828549.480478:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787828549.480482:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c7016580 x1874670646740992/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828574 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787828549.480495:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828549.480498:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787828549.480500:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828549.480502:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828549.480503:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828549.480505:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787828549.480506:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787828549.480509:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828549.480512:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:2.0:1787828549.480514:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1787828549.480517:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c7016580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646740992:192.168.203.143@tcp:38:kworker.0 00000100:00000001:2.0:1787828549.480522:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1787828549.480524:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787828549.480525:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828549.480528:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787828549.480531:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828549.480534:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787828549.480539:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c7015a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646741120:192.168.203.143@tcp:38:kworker.0 00000100:00000001:2.0:1787828549.480543:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787828549.480547:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1787828549.480549:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828549.480552:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1787828549.480556:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828549.480563:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787828549.480565:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369542208 : -126234340009408 : ffff8d30c7015a40) 00000100:00000001:2.0:1787828549.480570:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1787828549.480589:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:2.0:1787828549.480596:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1787828549.480600:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828549.480603:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828549.480606:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1787828549.480610:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828549.480616:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298022423 00000100:00000001:2.0:1787828549.480620:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298023681 00000100:00000001:2.0:1787828549.480622:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298022423 00000100:00000001:2.0:1787828549.480624:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298023681 00000100:00000001:2.0:1787828549.480626:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787828549.480629:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828549.480665:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1787828549.480672:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828549/real 1787828549] req@ffff8d30c7015a40 x1874670646741120/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828574 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1787828549.480698:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787828549.480700:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1787828549.480705:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1787828549.480708:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787828549.480710:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828549.480714:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828549.480716:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787828549.480722:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c7015a40 x1874670646741120/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828574 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787828549.480729:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828549.480731:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787828549.480734:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828549.480735:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828549.480736:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828549.480738:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787828549.480739:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787828549.480742:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828549.480745:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:2.0:1787828549.480747:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1787828549.480751:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c7015a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646741120:192.168.203.143@tcp:38:kworker.0 00000100:00000001:2.0:1787828549.480755:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1787828549.480756:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787828549.480758:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828549.480760:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787828549.480762:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828549.480764:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787828549.480767:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c7014f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646741248:192.168.203.143@tcp:8:kworker.0 00000100:00000001:2.0:1787828549.480770:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787828549.480773:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1787828549.480774:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828549.480776:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1787828549.480779:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828549.480785:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787828549.480787:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369539328 : -126234340012288 : ffff8d30c7014f00) 00000100:00000001:2.0:1787828549.480790:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1787828549.480804:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:2.0:1787828549.480809:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1787828549.480811:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828549.480812:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828549.480814:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1787828549.480817:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828549.480820:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298022437 00000100:00000001:2.0:1787828549.480823:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298023681 00000100:00000001:2.0:1787828549.480825:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298022437 00000100:00000001:2.0:1787828549.480827:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298023681 00000100:00000001:2.0:1787828549.480829:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787828549.480831:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828549.480833:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1787828549.480836:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828549/real 1787828549] req@ffff8d30c7014f00 x1874670646741248/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828574 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1787828549.480849:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787828549.480851:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1787828549.480853:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1787828549.480854:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787828549.480856:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828549.480857:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828549.480858:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787828549.480862:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c7014f00 x1874670646741248/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828574 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787828549.480875:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828549.480877:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787828549.480879:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828549.480881:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828549.480882:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828549.480884:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787828549.480885:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787828549.480887:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828549.480890:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:2.0:1787828549.480892:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1787828549.480894:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c7014f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646741248:192.168.203.143@tcp:8:kworker.0 00000100:00000001:2.0:1787828549.480898:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828549.480900:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828549.480901:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787828549.480902:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1787828549.480904:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787828549.480906:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:2.0:1787828549.480907:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:2.0:1787828549.480909:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787828549.480910:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1787828549.480912:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787828549.480913:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828549.480915:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828549.480916:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787828549.480918:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1787828549.480920:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787828549.480921:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:2.0:1787828549.480922:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:2.0:1787828549.480924:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787828549.480925:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1787828549.480926:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787828549.480927:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828549.480929:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828549.480930:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787828549.480931:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1787828549.480932:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787828549.480933:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:2.0:1787828549.480934:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:2.0:1787828549.480935:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787828549.480936:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1787828549.480938:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787828549.480939:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828549.480941:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828549.480944:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1787828549.480945:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828549.480947:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787828549.480949:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828550.374781:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828550.374789:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787828550.374912:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787828550.374918:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787828550.374922:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828551.398767:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828551.398777:0:815:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1787828551.398780:0:815:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1787828551.398800:0:815:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828551.398804:0:815:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1787828551.398806:0:815:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828551.398808:0:815:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828551.398809:0:815:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828551.398939:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828551.398949:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828551.398956:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828552.422751:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828552.422757:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787828552.422816:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787828552.422821:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787828552.422825:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828553.446672:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828553.446678:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787828553.446759:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787828553.446762:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787828553.446765:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828554.470751:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828554.470758:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828554.470882:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828554.470894:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828554.470903:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787828554.598860:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828554.598872:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1787828554.598875:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787828554.598880:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787828554.598886:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787828554.598895:0:40701:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 3355 00000100:00080000:2.0:1787828554.598908:0:40701:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787828554.598912:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828554.598915:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787828554.598921:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787828554.598924:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787828554.598928:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:2.0:1787828554.598931:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:2.0:1787828554.598934:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828554.598936:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:2.0:1787828554.598942:0:40701:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787828554.598945:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828554.598949:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787828554.598953:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828554.598957:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828554.598960:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:2.0:1787828554.598963:0:40701:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828554.598977:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787828554.598979:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828554.598983:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787828554.598986:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787828554.598992:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828554.599021:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787828554.599034:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2de00 x1874670646741376/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787828554.599047:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787828554.599052:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828554.599083:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787828554.599087:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828554.599092:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1787828554.599094:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787828554.599096:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787828554.599100:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787828554.599102:0:40701:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3355 00000100:00080000:2.0:1787828554.599106:0:40701:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787828554.599109:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828554.599113:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787828554.599117:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787828554.599119:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787828554.599122:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000020:00000001:2.0:1787828554.599127:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:2.0:1787828554.599132:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828554.599134:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:2.0:1787828554.599139:0:40701:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787828554.599142:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828554.599144:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:0.0:1787828554.599155:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:2.0:1787828554.599155:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828554.599166:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828554.599173:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1787828554.599173:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1787828554.599193:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000008:00100000:2.0:1787828554.599215:0:40701:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00000001:0.0:1787828554.599226:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000008:00000001:2.0:1787828554.599231:0:40701:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828554.599232:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828554.599238:0:40701:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828554.599247:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787828554.599249:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787828554.599260:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828554.599262:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787828554.599271:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787828554.599273:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828554.599275:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787828554.599283:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828554.599294:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646741376:192.168.203.143@tcp:250:kworker.0 00000100:00000001:2.0:1787828554.599298:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828554.599317:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00100000:2.0:1787828554.599325:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2fc00 x1874670646741504/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:0.0:1787828554.599329:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787828554.599338:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828554.599358:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 02000000:00000001:0.0:1787828554.599360:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:2.0:1787828554.599370:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828554.599379:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828554.599380:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787828554.599386:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828554.599394:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1787828554.599397:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787828554.599399:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787828554.599404:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787828554.599409:0:40701:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3355 00000100:00000001:0.0:1787828554.599414:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00080000:2.0:1787828554.599415:0:40701:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787828554.599421:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417875968 : -126234291675648 : ffff8d30c9e2de00) 00000100:00000001:2.0:1787828554.599423:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828554.599432:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:0.0:1787828554.599436:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000020:00000001:2.0:1787828554.599444:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787828554.599447:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787828554.599451:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000020:00000001:2.0:1787828554.599454:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:2.0:1787828554.599458:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828554.599459:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:2.0:1787828554.599465:0:40701:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787828554.599468:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828554.599470:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787828554.599476:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828554.599478:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828554.599480:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1787828554.599483:0:40701:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1787828554.599488:0:40701:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828554.599491:0:40701:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828554.599496:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787828554.599497:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828554.599501:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787828554.599502:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787828554.599505:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:0.0:1787828554.599518:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:2.0:1787828554.599523:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787828554.599535:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2d2c0 x1874670646741632/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787828554.599541:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1787828554.599552:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828554.599554:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787828554.599560:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828554.599560:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828554.599587:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1787828554.599590:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787828554.599598:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1787828554.599653:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828554.599662:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4298028801 00000100:00080000:2.0:1787828554.599674:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1787828554.599677:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787828554.599680:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787828554.599681:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 00000100:00000001:2.0:1787828554.599699:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787828554.599705:0:40701:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3355 00000100:00000001:0.0:1787828554.599707:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4298028801 00000100:00080000:2.0:1787828554.599717:0:40701:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787828554.599719:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 00000100:00000001:2.0:1787828554.599731:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787828554.599734:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787828554.599742:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:0.0:1787828554.599744:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828554.599751:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000020:00000001:2.0:1787828554.599762:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787828554.599768:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00100000:0.0:1787828554.599771:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828554/real 1787828554] req@ffff8d30c9e2de00 x1874670646741376/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828579 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:2.0:1787828554.599776:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000020:00000001:2.0:1787828554.599802:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000400:00000001:0.0:1787828554.599812:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000001:2.0:1787828554.599816:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000001:0.0:1787828554.599817:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1787828554.599826:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:0.0:1787828554.599830:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787828554.599838:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00080000:2.0:1787828554.599840:0:40701:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787828554.599845:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828554.599848:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828554.599854:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 02000000:00000001:2.0:1787828554.599855:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:0.0:1787828554.599870:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:2.0:1787828554.599873:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828554.599879:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828554.599884:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2de00 x1874670646741376/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828579 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787828554.599888:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1787828554.599898:0:40701:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000008:00000001:2.0:1787828554.599913:0:40701:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828554.599933:0:40701:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828554.599934:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828554.599963:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 02000000:00000001:2.0:1787828554.599964:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787828554.599969:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828554.599978:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:0.0:1787828554.599985:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:2.0:1787828554.599986:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787828554.599993:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828554.600001:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828554.600005:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828554.600009:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787828554.600019:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2cb40 x1874670646741760/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787828554.600021:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787828554.600031:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787828554.600039:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787828554.600048:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787828554.600052:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00000001:2.0:1787828554.600055:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787828554.600077:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787828554.600085:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787828554.600097:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646741376:192.168.203.143@tcp:250:kworker.0 00000100:00000001:0.0:1787828554.600110:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828554.600117:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828554.600121:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787828554.600127:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787828554.600134:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787828554.600139:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787828554.600144:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787828554.600151:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787828554.600155:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787828554.600167:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787828554.600169:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828554.600174:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828554.600189:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787828554.600193:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828554.600224:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787828554.600228:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787828554.600231:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1787828554.600234:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787828554.600237:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828554.600242:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787828554.600246:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787828554.600251:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828554.600258:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646741504:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787828554.600264:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787828554.600268:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787828554.600271:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828554.600275:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1787828554.600280:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828554.600292:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787828554.600295:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417883648 : -126234291667968 : ffff8d30c9e2fc00) 00000100:00000001:0.0:1787828554.600304:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1787828554.600332:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:0.0:1787828554.600341:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1787828554.600347:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828554.600350:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828554.600353:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787828554.600359:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828554.600368:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298027543 00000100:00000001:0.0:1787828554.600374:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298028801 00000100:00000001:0.0:1787828554.600379:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298027543 00000100:00000001:0.0:1787828554.600384:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298028801 00000100:00000001:0.0:1787828554.600388:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787828554.600392:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828554.600396:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787828554.600405:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828554/real 1787828554] req@ffff8d30c9e2fc00 x1874670646741504/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828579 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787828554.600418:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787828554.600420:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787828554.600424:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787828554.600426:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787828554.600428:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828554.600431:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828554.600433:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787828554.600438:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2fc00 x1874670646741504/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828579 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787828554.600448:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828554.600451:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787828554.600454:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828554.600456:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828554.600458:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828554.600460:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787828554.600463:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787828554.600466:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828554.600470:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787828554.600472:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787828554.600476:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646741504:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787828554.600482:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1787828554.600484:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787828554.600487:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828554.600490:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787828554.600493:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787828554.600495:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828554.600499:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2d2c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646741632:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787828554.600503:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787828554.600506:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787828554.600508:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828554.600511:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1787828554.600516:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828554.600523:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787828554.600525:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417873088 : -126234291678528 : ffff8d30c9e2d2c0) 00000100:00000001:0.0:1787828554.600530:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1787828554.600546:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:0.0:1787828554.600552:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1787828554.600555:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828554.600557:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828554.600559:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787828554.600563:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828554.600568:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298027543 00000100:00000001:0.0:1787828554.600571:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298028801 00000100:00000001:0.0:1787828554.600575:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298027543 00000100:00000001:0.0:1787828554.600577:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298028801 00000100:00000001:0.0:1787828554.600580:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787828554.600583:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828554.600585:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787828554.600589:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828554/real 1787828554] req@ffff8d30c9e2d2c0 x1874670646741632/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828579 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787828554.600630:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787828554.600632:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787828554.600635:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787828554.600637:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787828554.600639:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828554.600642:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828554.600644:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787828554.600649:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2d2c0 x1874670646741632/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828579 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787828554.600659:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828554.600662:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787828554.600665:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828554.600667:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828554.600668:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828554.600670:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787828554.600672:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787828554.600675:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828554.600679:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787828554.600681:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787828554.600685:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2d2c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646741632:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787828554.600689:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1787828554.600691:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787828554.600693:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828554.600696:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787828554.600698:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787828554.600701:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828554.600705:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646741760:192.168.203.143@tcp:8:kworker.0 00000100:00000001:0.0:1787828554.600709:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787828554.600712:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787828554.600714:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828554.600716:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1787828554.600720:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828554.600727:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787828554.600729:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417871168 : -126234291680448 : ffff8d30c9e2cb40) 00000100:00000001:0.0:1787828554.600733:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1787828554.600747:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:0.0:1787828554.600752:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1787828554.600755:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828554.600757:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828554.600759:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787828554.600762:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828554.600766:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298027557 00000100:00000001:0.0:1787828554.600770:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298028801 00000100:00000001:0.0:1787828554.600773:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298027557 00000100:00000001:0.0:1787828554.600776:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298028801 00000100:00000001:0.0:1787828554.600779:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787828554.600781:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828554.600783:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787828554.600788:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828554/real 1787828554] req@ffff8d30c9e2cb40 x1874670646741760/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828579 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787828554.600797:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787828554.600798:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787828554.600800:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787828554.600802:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787828554.600804:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828554.600806:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828554.600808:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787828554.600813:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2cb40 x1874670646741760/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828579 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787828554.600821:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828554.600823:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787828554.600826:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828554.600828:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828554.600829:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828554.600831:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787828554.600833:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787828554.600836:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828554.600841:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787828554.600844:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787828554.600848:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646741760:192.168.203.143@tcp:8:kworker.0 00000100:00000001:0.0:1787828554.600853:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828554.600855:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828554.600857:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787828554.600859:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787828554.600861:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787828554.600863:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787828554.600864:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787828554.600867:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787828554.600869:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787828554.600873:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787828554.600875:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828554.600877:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828554.600879:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787828554.600881:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787828554.600883:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787828554.600885:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787828554.600886:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787828554.600888:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787828554.600890:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787828554.600892:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787828554.600893:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828554.600895:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828554.600896:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787828554.600898:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787828554.600899:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787828554.600901:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787828554.600902:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787828554.600904:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787828554.600905:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787828554.600908:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787828554.600910:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828554.600912:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828554.600916:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787828554.600918:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828554.600920:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787828554.600923:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828555.494718:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828555.494724:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787828555.494882:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787828555.494891:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787828555.494898:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828556.518763:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828556.518771:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787828556.518873:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787828556.518881:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787828556.518888:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828557.542713:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828557.542720:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828557.542878:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828557.542901:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828557.542907:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828558.566712:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828558.566717:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787828558.566773:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787828558.566777:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787828558.566780:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828559.590695:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828559.590701:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787828559.590765:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787828559.590769:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787828559.590773:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787828559.718799:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828559.718816:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1787828559.718820:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787828559.718842:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787828559.718848:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787828559.718856:0:40701:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 3360 00000100:00080000:0.0:1787828559.718869:0:40701:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787828559.718873:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787828559.718876:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787828559.718882:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787828559.718885:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787828559.718889:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:0.0:1787828559.718893:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:0.0:1787828559.718896:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787828559.718898:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:0.0:1787828559.718921:0:40701:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787828559.718927:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828559.718931:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787828559.718937:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787828559.718942:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828559.718946:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1787828559.718951:0:40701:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828559.718967:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787828559.718970:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828559.718975:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787828559.718979:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787828559.718985:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828559.719024:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828559.719037:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d252c0 x1874670646741888/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787828559.719051:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787828559.719057:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828559.719098:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787828559.719104:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828559.719111:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1787828559.719116:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787828559.719119:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787828559.719126:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787828559.719132:0:40701:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3360 00000100:00000001:2.0:1787828559.719139:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00080000:0.0:1787828559.719141:0:40701:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787828559.719147:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787828559.719149:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828559.719152:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1787828559.719157:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:2.0:1787828559.719181:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787828559.719185:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787828559.719192:0:40701:0:(genops.c:792:class_conn2export()) Process entered 02000000:00000001:2.0:1787828559.719194:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000020:00000001:0.0:1787828559.719201:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 02000000:00000001:2.0:1787828559.719202:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828559.719218:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787828559.719236:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00100000:2.0:1787828559.719242:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d252c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646741888:192.168.203.143@tcp:250:kworker.0 00000020:00000001:0.0:1787828559.719244:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:2.0:1787828559.719252:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:0.0:1787828559.719253:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787828559.719257:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 02000000:00000001:2.0:1787828559.719258:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1787828559.719263:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828559.719266:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00080000:0.0:1787828559.719269:0:40701:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 02000000:00000001:2.0:1787828559.719273:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828559.719276:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828559.719282:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787828559.719289:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787828559.719290:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 02000000:00000001:0.0:1787828559.719295:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828559.719296:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601341120 : -126234108210496 : ffff8d30d4d252c0) 00000100:00000001:0.0:1787828559.719304:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:2.0:1787828559.719305:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000008:00100000:0.0:1787828559.719312:0:40701:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1787828559.719318:0:40701:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828559.719321:0:40701:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828559.719328:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787828559.719330:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828559.719334:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787828559.719336:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787828559.719339:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828559.719349:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828559.719356:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828559.719359:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d25680 x1874670646742016/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787828559.719362:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4298033920 00000100:00000001:2.0:1787828559.719375:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1787828559.719377:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787828559.719383:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828559.719384:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4298033920 00000100:00000001:2.0:1787828559.719392:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1787828559.719394:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828559.719397:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787828559.719402:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787828559.719403:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:2.0:1787828559.719405:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828559.719412:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:0.0:1787828559.719414:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1787828559.719428:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787828559.719431:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787828559.719432:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1787828559.719437:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:0.0:1787828559.719439:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:2.0:1787828559.719440:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787828559.719444:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00080000:0.0:1787828559.719448:0:40701:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3360 00000100:00000001:2.0:1787828559.719448:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1787828559.719453:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787828559.719456:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787828559.719458:0:40701:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 02000000:00000001:2.0:1787828559.719459:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787828559.719462:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787828559.719465:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 02000000:00000001:2.0:1787828559.719466:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828559.719472:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00100000:2.0:1787828559.719474:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d25680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646742016:192.168.203.143@tcp:38:kworker.0 00000020:00000001:0.0:1787828559.719479:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:2.0:1787828559.719481:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000020:00000001:0.0:1787828559.719484:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 02000000:00000001:2.0:1787828559.719485:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1787828559.719490:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787828559.719491:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 02000000:00000001:2.0:1787828559.719492:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000020:00000001:0.0:1787828559.719499:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 02000000:00000001:2.0:1787828559.719499:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828559.719507:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787828559.719510:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:2.0:1787828559.719511:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787828559.719514:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601342080 : -126234108209536 : ffff8d30d4d25680) 00000100:00000001:2.0:1787828559.719520:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00080000:0.0:1787828559.719524:0:40701:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787828559.719532:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828559.719533:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828559.719536:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:2.0:1787828559.719540:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298032662 02000000:00000001:0.0:1787828559.719544:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787828559.719545:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298033920 02000000:00000001:0.0:1787828559.719554:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828559.719554:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298032662 00000100:00000001:2.0:1787828559.719560:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298033920 00000100:00000001:0.0:1787828559.719562:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:2.0:1787828559.719564:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000008:00100000:0.0:1787828559.719568:0:40701:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00000001:2.0:1787828559.719569:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828559.719574:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:0.0:1787828559.719578:0:40701:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828559.719578:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828559.719585:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787828559.719586:0:40701:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828559.719588:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:2.0:1787828559.719592:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787828559.719593:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1787828559.719595:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828559.719597:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787828559.719603:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828559.719607:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787828559.719610:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787828559.719613:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828559.719621:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828559.719670:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828559.719681:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d26940 x1874670646742144/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787828559.719694:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787828559.719698:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828559.719722:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787828559.719726:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828559.719731:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1787828559.719734:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787828559.719737:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787828559.719741:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787828559.719746:0:40701:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3360 00000100:00000001:2.0:1787828559.719749:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00080000:0.0:1787828559.719753:0:40701:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787828559.719754:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1787828559.719757:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1787828559.719760:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787828559.719763:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828559.719763:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828559.719769:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 02000000:00000001:2.0:1787828559.719770:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787828559.719773:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000020:00000001:0.0:1787828559.719778:0:40701:0:(genops.c:792:class_conn2export()) Process entered 02000000:00000001:2.0:1787828559.719778:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787828559.719785:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00100000:2.0:1787828559.719787:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d26940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646742144:192.168.203.143@tcp:38:kworker.0 00000020:00000001:0.0:1787828559.719799:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:2.0:1787828559.719801:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787828559.719804:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000020:00000001:0.0:1787828559.719808:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 02000000:00000001:2.0:1787828559.719808:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828559.719814:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:0.0:1787828559.719818:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 02000000:00000001:2.0:1787828559.719820:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828559.719823:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:2.0:1787828559.719828:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787828559.719831:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601346880 : -126234108204736 : ffff8d30d4d26940) 00000100:00000001:2.0:1787828559.719837:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00080000:0.0:1787828559.719843:0:40701:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787828559.719851:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828559.719851:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828559.719860:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:2.0:1787828559.719860:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298032663 00000100:00000001:2.0:1787828559.719866:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298033920 02000000:00000001:0.0:1787828559.719868:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787828559.719869:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298032663 00000100:00000001:2.0:1787828559.719876:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298033920 02000000:00000001:0.0:1787828559.719877:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828559.719881:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787828559.719884:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:2.0:1787828559.719885:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000008:00100000:0.0:1787828559.719891:0:40701:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00000001:2.0:1787828559.719898:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:0.0:1787828559.719900:0:40701:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828559.719901:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828559.719908:0:40701:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828559.719909:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1787828559.719914:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:2.0:1787828559.719916:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787828559.719917:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 02000000:00000001:0.0:1787828559.719919:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:2.0:1787828559.719920:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828559.719926:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828559.719926:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828559.719933:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787828559.719935:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787828559.719938:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828559.719945:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828559.719954:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d27c00 x1874670646742272/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787828559.719964:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787828559.719967:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828559.720001:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828559.720050:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787828559.720054:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1787828559.720056:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1787828559.720058:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787828559.720061:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828559.720065:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787828559.720068:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828559.720071:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787828559.720077:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d27c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646742272:192.168.203.143@tcp:8:kworker.0 00000100:00000001:2.0:1787828559.720081:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787828559.720084:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1787828559.720086:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828559.720088:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1787828559.720093:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828559.720102:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787828559.720104:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601351680 : -126234108199936 : ffff8d30d4d27c00) 00000100:00000001:2.0:1787828559.720109:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:2.0:1787828559.720128:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828559.720133:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298032676 00000100:00000001:2.0:1787828559.720136:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298033920 00000100:00000001:2.0:1787828559.720138:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298032676 00000100:00000001:2.0:1787828559.720140:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298033920 00000100:00000001:2.0:1787828559.720142:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787828559.720145:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828559.720149:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828559.720151:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828559.720155:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1787828559.720157:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:2.0:1787828559.720181:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787828559.720182:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1787828559.720184:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828559.720187:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828560.614755:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828560.614780:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828560.614858:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828560.614864:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828560.614880:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828561.638706:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828561.638712:0:815:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1787828561.638714:0:815:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1787828561.638717:0:815:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828561.638720:0:815:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1787828561.638722:0:815:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828561.638724:0:815:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828561.638726:0:815:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787828561.638802:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787828561.638807:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787828561.638810:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828562.662691:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828562.662697:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787828562.662763:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787828562.662767:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787828562.662771:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:2.0:1787828562.790990:0:2379:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.143:988 00000400:00000100:2.0:1787828562.791048:0:2379:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.143@tcp at host 192.168.203.143 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1787828562.791076:0:2379:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.143@tcp 00000100:00000001:2.0:1787828562.791111:0:2379:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1787828562.791166:0:2379:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828562.791170:0:2379:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828562.791191:0:2379:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:2.0:1787828562.791203:0:2379:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.143@tcp 00000100:00000001:2.0:1787828562.791210:0:2379:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1787828562.791213:0:2379:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828562.791216:0:2379:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828562.791218:0:2379:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:2.0:1787828562.791224:0:2379:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.143@tcp 00000100:00000001:0.0:1787828562.791228:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787828562.791241:0:2379:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1787828562.791245:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1787828562.791247:0:2379:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828562.791252:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00000001:2.0:1787828562.791253:0:2379:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828562.791258:0:2379:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:2.0:1787828562.791268:0:2379:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.143@tcp 00000100:00100000:0.0:1787828562.791270:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828559/real 1787828562] req@ffff8d30d4d26940 x1874670646742144/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828584 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787828562.791274:0:2379:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1787828562.791279:0:2379:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828562.791283:0:2379:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828562.791288:0:2379:0:(events.c:68:request_out_callback()) Process leaving 00000400:00000001:0.0:1787828562.791298:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787828562.791302:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787828562.791322:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787828562.791325:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787828562.791327:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828562.791331:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828562.791333:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787828562.791340:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d26940 x1874670646742144/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828584 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787828562.791349:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828562.791352:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787828562.791356:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828562.791358:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828562.791359:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828562.791362:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787828562.791366:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787828562.791370:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828562.791376:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787828562.791379:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787828562.791384:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d26940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646742144:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787828562.791390:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787828562.791395:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828559/real 1787828562] req@ffff8d30d4d25680 x1874670646742016/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828584 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787828562.791406:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787828562.791407:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787828562.791411:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787828562.791413:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787828562.791414:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828562.791416:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828562.791418:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787828562.791423:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d25680 x1874670646742016/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828584 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787828562.791443:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828562.791445:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787828562.791447:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828562.791449:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828562.791451:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828562.791452:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787828562.791455:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787828562.791457:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828562.791462:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787828562.791464:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787828562.791468:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d25680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646742016:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787828562.791475:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787828562.791479:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828559/real 1787828562] req@ffff8d30d4d252c0 x1874670646741888/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828584 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787828562.791489:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787828562.791490:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787828562.791506:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787828562.791508:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787828562.791510:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828562.791512:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828562.791513:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787828562.791518:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d252c0 x1874670646741888/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828584 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787828562.791525:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828562.791527:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787828562.791530:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828562.791532:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828562.791534:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828562.791536:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787828562.791537:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787828562.791540:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828562.791544:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787828562.791546:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787828562.791550:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d252c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646741888:192.168.203.143@tcp:250:kworker.0 00000100:00000001:0.0:1787828562.791554:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828562.791556:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828562.791558:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787828562.791562:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787828562.791567:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787828562.791570:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787828562.791572:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787828562.791576:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787828562.791579:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787828562.791583:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787828562.791585:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828562.791587:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828562.791589:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787828562.791590:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787828562.791592:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787828562.791593:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787828562.791594:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787828562.791596:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787828562.791597:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787828562.791631:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787828562.791632:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828562.791635:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828562.791636:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787828562.791638:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787828562.791640:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787828562.791642:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787828562.791644:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787828562.791645:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787828562.791659:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787828562.791660:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787828562.791662:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828562.791665:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828562.791667:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787828562.791669:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787828562.791670:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787828562.791676:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828559/real 1787828562] req@ffff8d30d4d27c00 x1874670646742272/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828584 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787828562.791685:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787828562.791687:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787828562.791690:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787828562.791694:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787828562.791697:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828562.791700:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828562.791702:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787828562.791708:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d27c00 x1874670646742272/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828584 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787828562.791717:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828562.791720:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787828562.791724:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828562.791726:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828562.791729:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828562.791731:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787828562.791734:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787828562.791737:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828562.791741:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787828562.791744:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787828562.791748:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d27c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646742272:192.168.203.143@tcp:8:kworker.0 00000100:00000001:0.0:1787828562.791753:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828562.791755:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828562.791757:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787828562.791758:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787828562.791761:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787828562.791763:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787828562.791764:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787828562.791767:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787828562.791768:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787828562.791772:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787828562.791774:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828562.791777:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828562.791805:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787828562.791808:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828562.791810:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787828562.791812:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828563.686706:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828563.686713:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828563.686767:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828563.686770:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828563.686773:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828564.710671:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828564.710677:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787828564.710728:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787828564.710739:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787828564.710742:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787828564.838728:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828564.838737:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1787828564.838739:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787828564.838743:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787828564.838746:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787828564.838759:0:40701:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 3365 00000100:00080000:2.0:1787828564.838768:0:40701:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787828564.838770:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828564.838772:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787828564.838775:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787828564.838777:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787828564.838780:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:2.0:1787828564.838781:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:2.0:1787828564.838783:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828564.838784:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:2.0:1787828564.838787:0:40701:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787828564.838789:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828564.838791:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787828564.838794:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828564.838796:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828564.838798:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:2.0:1787828564.838800:0:40701:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828564.838807:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787828564.838809:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828564.838812:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787828564.838814:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787828564.838817:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828564.838826:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787828564.838834:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2e1c0 x1874670646742400/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787828564.838842:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787828564.838846:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828564.838857:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787828564.838859:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828564.838862:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1787828564.838864:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787828564.838866:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787828564.838868:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787828564.838871:0:40701:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3365 00000100:00080000:2.0:1787828564.838875:0:40701:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787828564.838878:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828564.838880:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787828564.838884:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787828564.838885:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787828564.838888:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000020:00000001:2.0:1787828564.838891:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:2.0:1787828564.838893:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828564.838895:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:0.0:1787828564.838896:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00080000:2.0:1787828564.838899:0:40701:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787828564.838903:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1787828564.838905:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828564.838908:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 02000000:00000001:2.0:1787828564.838909:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787828564.838917:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828564.838919:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828564.838921:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1787828564.838923:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000008:00100000:2.0:1787828564.838926:0:40701:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00000001:0.0:1787828564.838927:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:2.0:1787828564.838935:0:40701:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828564.838937:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:2.0:1787828564.838941:0:40701:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828564.838942:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787828564.838949:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828564.838950:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787828564.838955:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828564.838958:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646742400:192.168.203.143@tcp:250:kworker.0 00000100:00000001:2.0:1787828564.838959:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787828564.838963:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787828564.838967:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828564.838974:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:2.0:1787828564.838977:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828564.838981:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787828564.838985:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787828564.838988:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2d680 x1874670646742528/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:0.0:1787828564.838990:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1787828564.838997:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828564.839002:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787828564.839007:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828564.839010:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787828564.839014:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828564.839017:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1787828564.839019:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787828564.839021:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787828564.839024:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:0.0:1787828564.839025:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00080000:2.0:1787828564.839029:0:40701:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3365 00000100:00000001:0.0:1787828564.839030:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417876928 : -126234291674688 : ffff8d30c9e2e1c0) 00000100:00080000:2.0:1787828564.839037:0:40701:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787828564.839038:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:2.0:1787828564.839043:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828564.839046:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787828564.839050:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787828564.839051:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787828564.839054:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000020:00000001:2.0:1787828564.839056:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:2.0:1787828564.839059:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828564.839060:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:2.0:1787828564.839064:0:40701:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787828564.839068:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828564.839071:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787828564.839075:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828564.839078:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828564.839083:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000800:00000100:0.0:1787828564.839084:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000008:00100000:2.0:1787828564.839087:0:40701:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1787828564.839092:0:40701:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828564.839095:0:40701:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828564.839099:0:2390:0:(events.c:34:request_out_callback()) Process entered 02000000:00000001:2.0:1787828564.839102:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787828564.839105:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828564.839106:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828564.839109:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:0.0:1787828564.839111:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828564.839113:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:0.0:1787828564.839116:0:2390:0:(events.c:68:request_out_callback()) Process leaving 02000000:00000001:2.0:1787828564.839118:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828564.839139:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828564.839143:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787828564.839147:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2cb40 x1874670646742656/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787828564.839151:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4298039040 00000100:00000001:2.0:1787828564.839162:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787828564.839165:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828564.839166:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 00000100:00000001:2.0:1787828564.839168:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787828564.839174:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1787828564.839175:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4298039040 00000100:00080000:2.0:1787828564.839182:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1787828564.839185:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 00000100:00000001:2.0:1787828564.839186:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787828564.839190:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787828564.839193:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787828564.839196:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:0.0:1787828564.839199:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787828564.839203:0:40701:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3365 00000100:00000001:0.0:1787828564.839205:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00080000:2.0:1787828564.839211:0:40701:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1787828564.839217:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828564/real 1787828564] req@ffff8d30c9e2e1c0 x1874670646742400/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828589 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787828564.839218:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828564.839221:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787828564.839227:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787828564.839230:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787828564.839234:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000400:00000001:0.0:1787828564.839240:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000020:00000001:2.0:1787828564.839240:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000400:00000001:0.0:1787828564.839245:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1787828564.839246:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828564.839250:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:0.0:1787828564.839251:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787828564.839256:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00080000:2.0:1787828564.839258:0:40701:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787828564.839260:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828564.839264:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828564.839266:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 02000000:00000001:2.0:1787828564.839270:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:0.0:1787828564.839271:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:2.0:1787828564.839277:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828564.839282:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828564.839284:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00100000:0.0:1787828564.839288:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2e1c0 x1874670646742400/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828589 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:2.0:1787828564.839289:0:40701:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000008:00000001:2.0:1787828564.839295:0:40701:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828564.839300:0:40701:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828564.839308:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:0.0:1787828564.839310:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:2.0:1787828564.839312:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828564.839318:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787828564.839318:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787828564.839321:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:0.0:1787828564.839327:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:2.0:1787828564.839328:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828564.839334:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828564.839337:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828564.839338:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828564.839341:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787828564.839345:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00100000:2.0:1787828564.839349:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2d2c0 x1874670646742784/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828564.839352:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828564.839362:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787828564.839368:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000001:2.0:1787828564.839370:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787828564.839377:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828564.839378:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646742400:192.168.203.143@tcp:250:kworker.0 00000100:00000001:2.0:1787828564.839382:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828564.839387:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828564.839392:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828564.839394:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787828564.839397:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787828564.839402:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787828564.839404:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787828564.839407:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787828564.839412:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787828564.839415:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787828564.839420:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787828564.839422:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828564.839425:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828564.839432:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787828564.839434:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828564.839436:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787828564.839438:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787828564.839440:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1787828564.839441:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787828564.839443:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828564.839452:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787828564.839454:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787828564.839456:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828564.839460:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2d680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646742528:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787828564.839464:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787828564.839466:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787828564.839468:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828564.839470:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1787828564.839473:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828564.839480:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787828564.839481:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417874048 : -126234291677568 : ffff8d30c9e2d680) 00000100:00000001:0.0:1787828564.839487:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1787828564.839503:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:0.0:1787828564.839508:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1787828564.839511:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828564.839512:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828564.839514:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787828564.839517:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828564.839522:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298037782 00000100:00000001:0.0:1787828564.839525:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298039040 00000100:00000001:0.0:1787828564.839527:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298037782 00000100:00000001:0.0:1787828564.839530:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298039040 00000100:00000001:0.0:1787828564.839532:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787828564.839535:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828564.839537:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787828564.839541:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828564/real 1787828564] req@ffff8d30c9e2d680 x1874670646742528/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828589 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787828564.839552:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787828564.839553:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787828564.839556:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787828564.839558:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787828564.839559:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828564.839561:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828564.839563:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787828564.839567:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2d680 x1874670646742528/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828589 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787828564.839575:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828564.839577:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787828564.839580:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828564.839582:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828564.839583:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828564.839585:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787828564.839587:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787828564.839590:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828564.839593:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787828564.839595:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787828564.839624:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2d680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646742528:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787828564.839630:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1787828564.839632:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787828564.839633:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828564.839635:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787828564.839637:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787828564.839639:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828564.839643:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646742656:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787828564.839646:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787828564.839648:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787828564.839650:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828564.839652:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1787828564.839655:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828564.839660:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787828564.839662:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417871168 : -126234291680448 : ffff8d30c9e2cb40) 00000100:00000001:0.0:1787828564.839665:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1787828564.839677:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:0.0:1787828564.839682:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1787828564.839684:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828564.839685:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828564.839687:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787828564.839690:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828564.839694:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298037783 00000100:00000001:0.0:1787828564.839696:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298039040 00000100:00000001:0.0:1787828564.839699:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298037783 00000100:00000001:0.0:1787828564.839702:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298039040 00000100:00000001:0.0:1787828564.839704:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787828564.839706:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828564.839709:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787828564.839713:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828564/real 1787828564] req@ffff8d30c9e2cb40 x1874670646742656/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828589 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787828564.839723:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787828564.839724:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787828564.839727:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787828564.839729:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787828564.839731:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828564.839733:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828564.839734:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787828564.839738:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2cb40 x1874670646742656/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828589 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787828564.839747:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828564.839749:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787828564.839752:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828564.839754:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828564.839755:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828564.839757:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787828564.839759:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787828564.839762:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828564.839766:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787828564.839768:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787828564.839772:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646742656:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787828564.839781:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1787828564.839782:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787828564.839784:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828564.839786:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787828564.839788:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787828564.839790:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828564.839793:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2d2c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646742784:192.168.203.143@tcp:8:kworker.0 00000100:00000001:0.0:1787828564.839797:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787828564.839799:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787828564.839800:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828564.839802:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1787828564.839806:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828564.839812:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787828564.839814:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417873088 : -126234291678528 : ffff8d30c9e2d2c0) 00000100:00000001:0.0:1787828564.839817:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1787828564.839831:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:0.0:1787828564.839836:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1787828564.839839:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828564.839841:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828564.839842:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787828564.839846:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828564.839849:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298037796 00000100:00000001:0.0:1787828564.839852:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298039040 00000100:00000001:0.0:1787828564.839855:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298037796 00000100:00000001:0.0:1787828564.839857:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298039040 00000100:00000001:0.0:1787828564.839860:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787828564.839862:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828564.839864:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787828564.839869:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828564/real 1787828564] req@ffff8d30c9e2d2c0 x1874670646742784/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828589 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787828564.839878:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787828564.839879:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787828564.839882:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787828564.839884:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787828564.839886:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828564.839888:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828564.839890:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787828564.839894:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2d2c0 x1874670646742784/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828589 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787828564.839902:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828564.839904:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787828564.839907:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828564.839909:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828564.839910:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828564.839912:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787828564.839915:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787828564.839917:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828564.839921:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787828564.839923:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787828564.839927:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2d2c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646742784:192.168.203.143@tcp:8:kworker.0 00000100:00000001:0.0:1787828564.839931:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828564.839933:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828564.839934:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787828564.839936:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787828564.839939:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787828564.839940:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787828564.839942:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787828564.839945:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787828564.839946:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787828564.839950:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787828564.839952:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828564.839954:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828564.839955:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787828564.839957:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787828564.839958:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787828564.839960:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787828564.839961:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787828564.839963:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787828564.839965:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787828564.839967:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787828564.839969:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828564.839971:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828564.839972:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787828564.839974:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787828564.839976:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787828564.839977:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787828564.839979:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787828564.839981:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787828564.839982:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787828564.839984:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787828564.839985:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828564.839987:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828564.839991:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787828564.839993:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828564.839995:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787828564.839997:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828565.734716:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828565.734724:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787828565.734852:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787828565.734857:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787828565.734861:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828566.758730:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828566.758739:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828566.758814:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828566.758818:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828566.758822:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828567.782679:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828567.782684:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787828567.782745:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787828567.782749:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787828567.782753:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828568.806720:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828568.806727:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787828568.806807:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787828568.806812:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787828568.806817:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828569.830696:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828569.830701:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828569.830845:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828569.830851:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828569.830855:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787828569.958730:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828569.958741:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1787828569.958760:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787828569.958765:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787828569.958770:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787828569.958777:0:468:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 3370 00000100:00080000:3.0:1787828569.958790:0:468:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787828569.958793:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787828569.958795:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:3.0:1787828569.958799:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1787828569.958810:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1787828569.958830:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:3.0:1787828569.958833:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:3.0:1787828569.958835:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787828569.958836:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:3.0:1787828569.958841:0:468:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:3.0:1787828569.958843:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787828569.958846:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787828569.958850:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787828569.958853:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828569.958856:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:3.0:1787828569.958859:0:468:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787828569.958873:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787828569.958875:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828569.958878:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787828569.958881:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787828569.958891:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828569.958906:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787828569.958918:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c70143c0 x1874670646742912/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787828569.958941:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787828569.958948:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828569.958969:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787828569.958974:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828569.958978:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1787828569.958980:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787828569.958982:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787828569.958985:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787828569.959005:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3370 00000100:00080000:3.0:1787828569.959011:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787828569.959015:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787828569.959018:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:3.0:1787828569.959022:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1787828569.959024:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1787828569.959026:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:0.0:1787828569.959029:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:3.0:1787828569.959031:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:0.0:1787828569.959037:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787828569.959038:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787828569.959043:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787828569.959044:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:0.0:1787828569.959048:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787828569.959052:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787828569.959055:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 02000000:00000001:0.0:1787828569.959060:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:3.0:1787828569.959062:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828569.959065:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787828569.959068:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787828569.959071:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828569.959108:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c70143c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646742912:192.168.203.143@tcp:250:kworker.0 02000000:00000001:3.0:1787828569.959110:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787828569.959117:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1787828569.959119:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828569.959124:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 02000000:00000001:0.0:1787828569.959126:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000008:00100000:3.0:1787828569.959130:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 02000000:00000001:0.0:1787828569.959132:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:3.0:1787828569.959137:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828569.959139:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:3.0:1787828569.959144:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828569.959150:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787828569.959153:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787828569.959157:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828569.959160:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787828569.959162:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787828569.959164:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828569.959174:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787828569.959174:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828569.959178:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369536448 : -126234340015168 : ffff8d30c70143c0) 00000100:00000001:0.0:1787828569.959183:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00100000:3.0:1787828569.959187:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c7015680 x1874670646743040/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787828569.959199:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787828569.959202:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828569.959207:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787828569.959210:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828569.959214:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1787828569.959216:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787828569.959217:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787828569.959220:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787828569.959223:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3370 00000100:00080000:3.0:1787828569.959227:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1787828569.959233:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:3.0:1787828569.959233:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787828569.959237:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:3.0:1787828569.959241:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1787828569.959242:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1787828569.959244:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000020:00000001:3.0:1787828569.959247:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:3.0:1787828569.959248:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787828569.959250:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787828569.959250:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:3.0:1787828569.959257:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787828569.959258:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787828569.959262:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828569.959263:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787828569.959266:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:0.0:1787828569.959267:0:2390:0:(events.c:68:request_out_callback()) Process leaving 02000000:00000001:3.0:1787828569.959270:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787828569.959274:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828569.959275:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828569.959277:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:3.0:1787828569.959282:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00000001:0.0:1787828569.959283:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4298044160 00000008:00000001:3.0:1787828569.959285:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828569.959291:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787828569.959295:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787828569.959296:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828569.959298:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787828569.959299:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787828569.959301:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828569.959302:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 00000100:00000001:3.0:1787828569.959306:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787828569.959312:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c70152c0 x1874670646743168/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787828569.959318:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787828569.959320:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828569.959322:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4298044160 00000100:00000001:3.0:1787828569.959323:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787828569.959329:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828569.959333:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1787828569.959334:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:0.0:1787828569.959335:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 00000100:00080000:3.0:1787828569.959336:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787828569.959343:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:0.0:1787828569.959346:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00080000:3.0:1787828569.959347:0:468:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3370 00000100:00000001:0.0:1787828569.959353:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787828569.959354:0:468:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787828569.959358:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00000001:3.0:1787828569.959359:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787828569.959363:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:3.0:1787828569.959365:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1787828569.959366:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00100000:0.0:1787828569.959371:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828569/real 1787828569] req@ffff8d30c70143c0 x1874670646742912/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828594 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:3.0:1787828569.959371:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000020:00000001:3.0:1787828569.959375:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:3.0:1787828569.959380:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787828569.959383:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:3.0:1787828569.959391:0:468:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000400:00000001:0.0:1787828569.959394:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000001:3.0:1787828569.959394:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:0.0:1787828569.959398:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 02000000:00000001:3.0:1787828569.959399:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787828569.959403:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787828569.959404:0:2390:0:(events.c:78:reply_in_callback()) Process entered 02000000:00000001:3.0:1787828569.959405:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828569.959409:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787828569.959409:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1787828569.959414:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000008:00100000:3.0:1787828569.959415:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00000001:0.0:1787828569.959421:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000008:00000001:3.0:1787828569.959422:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828569.959426:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787828569.959427:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787828569.959436:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00100000:0.0:1787828569.959437:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c70143c0 x1874670646742912/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828594 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1787828569.959438:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828569.959443:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787828569.959445:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787828569.959448:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828569.959455:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787828569.959455:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828569.959461:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00100000:3.0:1787828569.959464:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c7014b40 x1874670646743296/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787828569.959465:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828569.959471:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828569.959473:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828569.959478:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787828569.959481:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787828569.959484:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828569.959489:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787828569.959491:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787828569.959492:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000001:3.0:1787828569.959495:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828569.959500:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c70143c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646742912:192.168.203.143@tcp:250:kworker.0 00000100:00000001:3.0:1787828569.959501:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828569.959510:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828569.959514:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828569.959515:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787828569.959518:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787828569.959521:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787828569.959523:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787828569.959525:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787828569.959530:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787828569.959532:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787828569.959534:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787828569.959536:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828569.959539:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828569.959548:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787828569.959550:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828569.959553:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787828569.959555:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787828569.959557:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1787828569.959559:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787828569.959560:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828569.959563:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787828569.959565:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787828569.959568:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828569.959573:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c7015680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646743040:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787828569.959577:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787828569.959580:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787828569.959582:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828569.959584:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1787828569.959588:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828569.959596:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787828569.959597:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369541248 : -126234340010368 : ffff8d30c7015680) 00000100:00000001:0.0:1787828569.959635:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1787828569.959648:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:0.0:1787828569.959653:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1787828569.959655:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828569.959656:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828569.959674:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787828569.959677:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828569.959682:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298042903 00000100:00000001:0.0:1787828569.959684:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298044160 00000100:00000001:0.0:1787828569.959687:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298042903 00000100:00000001:0.0:1787828569.959689:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298044160 00000100:00000001:0.0:1787828569.959691:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787828569.959693:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828569.959695:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787828569.959699:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828569/real 1787828569] req@ffff8d30c7015680 x1874670646743040/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828594 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787828569.959706:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787828569.959707:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787828569.959709:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787828569.959710:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787828569.959712:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828569.959714:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828569.959715:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787828569.959718:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c7015680 x1874670646743040/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828594 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787828569.959741:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828569.959743:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787828569.959745:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828569.959746:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828569.959747:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828569.959748:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787828569.959750:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787828569.959751:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828569.959753:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787828569.959755:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787828569.959757:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c7015680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646743040:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787828569.959760:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1787828569.959761:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787828569.959763:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828569.959766:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787828569.959767:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787828569.959770:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828569.959774:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c70152c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646743168:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787828569.959777:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787828569.959780:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787828569.959782:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828569.959784:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1787828569.959789:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828569.959829:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787828569.959831:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369540288 : -126234340011328 : ffff8d30c70152c0) 00000100:00000001:0.0:1787828569.959836:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1787828569.959852:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:0.0:1787828569.959873:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1787828569.959876:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828569.959878:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828569.959880:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787828569.959884:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828569.959899:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298042903 00000100:00000001:0.0:1787828569.959903:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298044160 00000100:00000001:0.0:1787828569.959907:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298042903 00000100:00000001:0.0:1787828569.959910:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298044160 00000100:00000001:0.0:1787828569.959912:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787828569.959915:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828569.959918:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787828569.959925:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828569/real 1787828569] req@ffff8d30c70152c0 x1874670646743168/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828594 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787828569.959952:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787828569.959954:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787828569.959958:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787828569.959960:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787828569.959961:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828569.959963:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828569.959965:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787828569.959968:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c70152c0 x1874670646743168/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828594 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787828569.959975:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828569.959977:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787828569.959978:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828569.959980:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828569.959981:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828569.959982:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787828569.959983:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787828569.959985:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828569.959988:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787828569.959989:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787828569.959992:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c70152c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646743168:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787828569.960011:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1787828569.960013:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787828569.960014:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828569.960016:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787828569.960018:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787828569.960020:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828569.960023:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c7014b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646743296:192.168.203.143@tcp:8:kworker.0 00000100:00000001:0.0:1787828569.960026:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787828569.960028:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787828569.960029:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828569.960031:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1787828569.960034:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828569.960040:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787828569.960041:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369538368 : -126234340013248 : ffff8d30c7014b40) 00000100:00000001:0.0:1787828569.960045:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1787828569.960057:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:0.0:1787828569.960061:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1787828569.960063:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828569.960064:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828569.960066:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787828569.960068:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828569.960071:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298042916 00000100:00000001:0.0:1787828569.960074:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298044160 00000100:00000001:0.0:1787828569.960108:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298042916 00000100:00000001:0.0:1787828569.960112:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298044160 00000100:00000001:0.0:1787828569.960114:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787828569.960117:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828569.960119:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787828569.960125:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828569/real 1787828569] req@ffff8d30c7014b40 x1874670646743296/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828594 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787828569.960135:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787828569.960137:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787828569.960140:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787828569.960142:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787828569.960144:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828569.960146:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828569.960148:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787828569.960153:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c7014b40 x1874670646743296/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828594 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787828569.960161:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828569.960163:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787828569.960165:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828569.960166:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828569.960167:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828569.960168:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787828569.960170:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787828569.960171:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828569.960174:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787828569.960176:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787828569.960179:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c7014b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646743296:192.168.203.143@tcp:8:kworker.0 00000100:00000001:0.0:1787828569.960181:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828569.960183:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828569.960184:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787828569.960185:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787828569.960187:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787828569.960188:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787828569.960189:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787828569.960191:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787828569.960192:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787828569.960194:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787828569.960195:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828569.960196:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828569.960197:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787828569.960198:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787828569.960200:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787828569.960201:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787828569.960202:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787828569.960203:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787828569.960204:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787828569.960204:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787828569.960205:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828569.960206:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828569.960207:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787828569.960208:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787828569.960209:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787828569.960210:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787828569.960211:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787828569.960212:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787828569.960212:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787828569.960214:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787828569.960214:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828569.960216:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828569.960218:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787828569.960219:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828569.960221:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787828569.960222:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828570.854734:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828570.854741:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787828570.854808:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787828570.854814:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787828570.854818:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828571.878697:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828571.878705:0:815:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1787828571.878708:0:815:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1787828571.878712:0:815:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828571.878716:0:815:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1787828571.878719:0:815:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828571.878721:0:815:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828571.878723:0:815:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787828571.878802:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787828571.878809:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787828571.878814:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828572.902719:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828572.902725:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828572.902773:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828572.902779:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828572.902783:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828573.926716:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828573.926723:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787828573.926906:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787828573.926917:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787828573.926927:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828574.950730:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828574.950737:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787828574.950837:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787828574.950842:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787828574.950846:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787828575.078711:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828575.078719:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1787828575.078722:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1787828575.078726:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787828575.078730:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1787828575.078737:0:468:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 3375 00000100:00080000:1.0:1787828575.078747:0:468:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1787828575.078750:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787828575.078751:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:1.0:1787828575.078755:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:1.0:1787828575.078757:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:1.0:1787828575.078760:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:1.0:1787828575.078762:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:1.0:1787828575.078764:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787828575.078765:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:1.0:1787828575.078769:0:468:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:1.0:1787828575.078771:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828575.078774:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1787828575.078777:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787828575.078781:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828575.078784:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:1.0:1787828575.078786:0:468:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828575.078804:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1787828575.078806:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828575.078810:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1787828575.078812:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1787828575.078816:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828575.078827:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787828575.078836:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d13ecb40 x1874670646743424/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787828575.078846:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787828575.078849:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828575.078862:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1787828575.078865:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828575.078867:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1787828575.078869:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1787828575.078870:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787828575.078873:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1787828575.078876:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3375 00000100:00080000:1.0:1787828575.078880:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1787828575.078882:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787828575.078884:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:1.0:1787828575.078886:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:1.0:1787828575.078887:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:0.0:1787828575.078889:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:1.0:1787828575.078890:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000020:00000001:1.0:1787828575.078895:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:0.0:1787828575.078896:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1787828575.078897:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787828575.078901:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:0.0:1787828575.078902:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00080000:1.0:1787828575.078907:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787828575.078908:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787828575.078911:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828575.078912:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828575.078916:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787828575.078919:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787828575.078925:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787828575.078925:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787828575.078932:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828575.078934:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828575.078937:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00100000:0.0:1787828575.078942:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d13ecb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646743424:192.168.203.143@tcp:250:kworker.0 00000008:00100000:1.0:1787828575.078942:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1787828575.078949:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828575.078952:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:1.0:1787828575.078953:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828575.078959:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787828575.078960:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787828575.078963:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828575.078963:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828575.078970:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787828575.078971:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787828575.078972:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1787828575.078976:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828575.078978:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828575.078983:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787828575.078990:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d13ec000 x1874670646743552/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787828575.078994:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787828575.078999:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839541340992 : -126234168210624 : ffff8d30d13ecb40) 00000100:00000001:1.0:1787828575.079000:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787828575.079006:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:1.0:1787828575.079006:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828575.079013:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1787828575.079017:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828575.079020:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:1.0:1787828575.079022:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1787828575.079024:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787828575.079026:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1787828575.079028:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3375 00000100:00080000:1.0:1787828575.079041:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1787828575.079043:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787828575.079044:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:1.0:1787828575.079046:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:1.0:1787828575.079047:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:1.0:1787828575.079048:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000020:00000001:1.0:1787828575.079050:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:1.0:1787828575.079066:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787828575.079069:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000800:00000100:0.0:1787828575.079071:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00080000:1.0:1787828575.079075:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:1.0:1787828575.079080:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828575.079082:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1787828575.079085:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787828575.079087:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828575.079089:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787828575.079090:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:1.0:1787828575.079095:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1787828575.079098:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828575.079100:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787828575.079101:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828575.079106:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828575.079107:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:0.0:1787828575.079111:0:2390:0:(events.c:68:request_out_callback()) Process leaving 02000000:00000001:1.0:1787828575.079112:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828575.079116:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1787828575.079135:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1787828575.079138:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828575.079140:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828575.079144:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828575.079146:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4298049280 00000100:00100000:1.0:1787828575.079154:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d13ec780 x1874670646743680/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787828575.079158:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1787828575.079169:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4298049280 00000100:00000001:1.0:1787828575.079171:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787828575.079174:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 00000100:00000001:1.0:1787828575.079177:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828575.079179:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787828575.079184:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828575.079187:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787828575.079194:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828575/real 1787828575] req@ffff8d30d13ecb40 x1874670646743424/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828600 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787828575.079195:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1787828575.079204:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828575.079215:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000400:00000001:0.0:1787828575.079219:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000001:1.0:1787828575.079219:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000400:00000001:0.0:1787828575.079225:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00080000:1.0:1787828575.079225:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787828575.079232:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:0.0:1787828575.079233:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00080000:1.0:1787828575.079238:0:468:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3375 00000100:00000001:0.0:1787828575.079240:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00080000:1.0:1787828575.079243:0:468:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787828575.079244:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828575.079251:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787828575.079252:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787828575.079256:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:0.0:1787828575.079257:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:1.0:1787828575.079263:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000100:00100000:0.0:1787828575.079268:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d13ecb40 x1874670646743424/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828600 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:1.0:1787828575.079274:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:1.0:1787828575.079279:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000020:00000001:1.0:1787828575.079286:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:0.0:1787828575.079288:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828575.079289:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787828575.079292:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:0.0:1787828575.079315:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787828575.079315:0:468:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:1.0:1787828575.079320:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828575.079324:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:0.0:1787828575.079326:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:1.0:1787828575.079328:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787828575.079333:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 02000000:00000001:1.0:1787828575.079333:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828575.079340:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828575.079340:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:1.0:1787828575.079343:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00000001:0.0:1787828575.079348:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000008:00000001:1.0:1787828575.079349:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828575.079354:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828575.079355:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787828575.079377:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 02000000:00000001:1.0:1787828575.079380:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1787828575.079384:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787828575.079385:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787828575.079388:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1787828575.079390:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1787828575.079392:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828575.079398:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828575.079405:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787828575.079407:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d13ee940 x1874670646743808/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787828575.079416:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d13ecb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646743424:192.168.203.143@tcp:250:kworker.0 00000100:00000001:1.0:1787828575.079419:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787828575.079425:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828575.079427:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828575.079429:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828575.079431:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828575.079436:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787828575.079439:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787828575.079444:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787828575.079447:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787828575.079450:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787828575.079454:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787828575.079457:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787828575.079461:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787828575.079463:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828575.079466:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828575.079474:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787828575.079476:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828575.079480:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787828575.079484:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787828575.079486:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1787828575.079490:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787828575.079492:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828575.079495:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787828575.079497:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787828575.079500:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828575.079503:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d13ec000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646743552:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787828575.079506:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787828575.079509:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787828575.079512:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828575.079514:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1787828575.079519:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828575.079529:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787828575.079532:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839541338112 : -126234168213504 : ffff8d30d13ec000) 00000100:00000001:0.0:1787828575.079554:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1787828575.079572:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:0.0:1787828575.079579:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1787828575.079583:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828575.079585:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828575.079587:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787828575.079591:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828575.079596:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298048022 00000100:00000001:0.0:1787828575.079599:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298049280 00000100:00000001:0.0:1787828575.079601:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298048022 00000100:00000001:0.0:1787828575.079603:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298049280 00000100:00000001:0.0:1787828575.079605:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787828575.079607:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828575.079608:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787828575.079612:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828575/real 1787828575] req@ffff8d30d13ec000 x1874670646743552/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828600 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787828575.079665:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787828575.079667:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787828575.079671:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787828575.079674:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787828575.079676:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828575.079678:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828575.079679:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787828575.079684:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d13ec000 x1874670646743552/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828600 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787828575.079690:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828575.079693:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787828575.079696:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828575.079698:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828575.079700:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828575.079702:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787828575.079704:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787828575.079708:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828575.079713:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787828575.079715:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787828575.079720:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d13ec000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646743552:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787828575.079726:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1787828575.079729:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787828575.079730:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828575.079732:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787828575.079734:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787828575.079737:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828575.079740:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d13ec780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646743680:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787828575.079743:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787828575.079747:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787828575.079748:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828575.079750:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1787828575.079753:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828575.079757:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787828575.079759:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839541340032 : -126234168211584 : ffff8d30d13ec780) 00000100:00000001:0.0:1787828575.079762:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1787828575.079774:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:0.0:1787828575.079777:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1787828575.079779:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828575.079781:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828575.079783:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787828575.079785:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828575.079788:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298048023 00000100:00000001:0.0:1787828575.079790:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298049280 00000100:00000001:0.0:1787828575.079792:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298048023 00000100:00000001:0.0:1787828575.079794:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298049280 00000100:00000001:0.0:1787828575.079796:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787828575.079798:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828575.079799:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787828575.079802:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828575/real 1787828575] req@ffff8d30d13ec780 x1874670646743680/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828600 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787828575.079808:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787828575.079809:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787828575.079811:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787828575.079813:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787828575.079814:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828575.079816:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828575.079817:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787828575.079820:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d13ec780 x1874670646743680/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828600 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787828575.079825:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828575.079827:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787828575.079829:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828575.079830:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828575.079831:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828575.079833:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787828575.079834:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787828575.079836:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828575.079838:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787828575.079840:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787828575.079843:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d13ec780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646743680:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787828575.079846:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1787828575.079847:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787828575.079848:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828575.079850:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787828575.079851:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787828575.079853:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828575.079855:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d13ee940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646743808:192.168.203.143@tcp:8:kworker.0 00000100:00000001:0.0:1787828575.079857:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787828575.079859:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787828575.079860:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828575.079861:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1787828575.079863:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828575.079867:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787828575.079868:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839541348672 : -126234168202944 : ffff8d30d13ee940) 00000100:00000001:0.0:1787828575.079870:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1787828575.079877:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:0.0:1787828575.079880:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1787828575.079882:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828575.079883:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828575.079884:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787828575.079886:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828575.079888:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298048036 00000100:00000001:0.0:1787828575.079891:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298049280 00000100:00000001:0.0:1787828575.079892:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298048036 00000100:00000001:0.0:1787828575.079894:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298049280 00000100:00000001:0.0:1787828575.079896:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787828575.079898:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828575.079899:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787828575.079902:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828575/real 1787828575] req@ffff8d30d13ee940 x1874670646743808/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828600 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787828575.079909:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787828575.079910:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787828575.079912:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787828575.079913:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787828575.079914:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828575.079915:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828575.079916:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787828575.079919:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d13ee940 x1874670646743808/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828600 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787828575.079924:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828575.079926:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787828575.079928:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828575.079929:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828575.079930:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828575.079931:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787828575.079933:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787828575.079935:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828575.079937:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787828575.079938:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787828575.079941:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d13ee940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646743808:192.168.203.143@tcp:8:kworker.0 00000100:00000001:0.0:1787828575.079944:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828575.079946:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828575.079947:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787828575.079948:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787828575.079950:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787828575.079952:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787828575.079953:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787828575.079955:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787828575.079956:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787828575.079958:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787828575.079959:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828575.079961:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828575.079962:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787828575.079963:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787828575.079964:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787828575.079965:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787828575.079966:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787828575.079968:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787828575.079969:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787828575.079970:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787828575.079971:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828575.079972:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828575.079973:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787828575.079974:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787828575.079975:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787828575.079976:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787828575.079976:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787828575.079978:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787828575.079978:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787828575.079980:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787828575.079981:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828575.079983:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828575.079986:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787828575.079987:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828575.079988:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787828575.079990:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828575.974860:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828575.974869:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828575.975011:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828575.975020:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828575.975026:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828576.998720:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828576.998726:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787828576.998780:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787828576.998785:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787828576.998788:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828578.022693:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828578.022699:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787828578.022784:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787828578.022791:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787828578.022796:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828579.046730:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828579.046739:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828579.046870:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828579.046877:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828579.046882:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828580.070790:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828580.070798:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787828580.070945:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787828580.070954:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787828580.070960:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787828580.198807:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828580.198827:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1787828580.198832:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1787828580.198840:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787828580.198850:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1787828580.198865:0:468:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 3380 00000100:00080000:1.0:1787828580.198884:0:468:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1787828580.198890:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787828580.198894:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:1.0:1787828580.198902:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:1.0:1787828580.198907:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:1.0:1787828580.198913:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:1.0:1787828580.198917:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:1.0:1787828580.198921:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787828580.198924:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:1.0:1787828580.198933:0:468:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:1.0:1787828580.198938:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828580.198943:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1787828580.198950:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787828580.198957:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828580.198962:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:1.0:1787828580.198967:0:468:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828580.198988:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1787828580.198992:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828580.198998:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1787828580.199003:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1787828580.199012:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828580.199063:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787828580.199085:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d13ee940 x1874670646743936/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787828580.199104:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787828580.199112:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828580.199142:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1787828580.199148:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828580.199154:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1787828580.199158:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1787828580.199161:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787828580.199167:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1787828580.199172:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3380 00000100:00080000:1.0:1787828580.199181:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1787828580.199187:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787828580.199190:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:1.0:1787828580.199196:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:1.0:1787828580.199199:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:1.0:1787828580.199204:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000020:00000001:1.0:1787828580.199209:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:1.0:1787828580.199213:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787828580.199217:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:0.0:1787828580.199222:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00080000:1.0:1787828580.199225:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787828580.199232:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1787828580.199235:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828580.199238:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1787828580.199244:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 02000000:00000001:1.0:1787828580.199247:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:0.0:1787828580.199249:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828580.199259:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787828580.199263:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787828580.199264:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787828580.199267:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828580.199276:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828580.199283:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d13ee940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646743936:192.168.203.143@tcp:250:kworker.0 00000100:00000001:0.0:1787828580.199292:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787828580.199301:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:1.0:1787828580.199301:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 02000000:00000001:0.0:1787828580.199310:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000008:00100000:1.0:1787828580.199315:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 02000000:00000001:0.0:1787828580.199316:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000008:00000001:1.0:1787828580.199326:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828580.199327:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828580.199333:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828580.199346:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1787828580.199349:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828580.199353:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1787828580.199356:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:0.0:1787828580.199357:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 02000000:00000001:1.0:1787828580.199362:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828580.199363:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839541348672 : -126234168202944 : ffff8d30d13ee940) 00000100:00000001:0.0:1787828580.199376:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:1.0:1787828580.199383:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787828580.199399:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d13ec780 x1874670646744064/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787828580.199419:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787828580.199424:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828580.199431:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:0.0:1787828580.199440:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00080000:1.0:1787828580.199442:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828580.199453:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:1.0:1787828580.199457:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:0.0:1787828580.199460:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00080000:1.0:1787828580.199462:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787828580.199469:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787828580.199471:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:0.0:1787828580.199475:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828580.199481:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00080000:1.0:1787828580.199484:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3380 00000100:00000001:0.0:1787828580.199492:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1787828580.199496:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787828580.199500:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4298054400 00000100:00000001:1.0:1787828580.199504:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787828580.199511:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:1.0:1787828580.199517:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:0.0:1787828580.199518:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 00000020:00000001:1.0:1787828580.199524:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:1.0:1787828580.199532:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:0.0:1787828580.199533:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4298054400 00000020:00000001:1.0:1787828580.199543:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:0.0:1787828580.199545:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1787828580.199554:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787828580.199555:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787828580.199560:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828580.199563:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:0.0:1787828580.199567:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00080000:1.0:1787828580.199576:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00100000:0.0:1787828580.199583:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828580/real 1787828580] req@ffff8d30d13ee940 x1874670646743936/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828605 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787828580.199587:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828580.199598:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1787828580.199650:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787828580.199656:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828580.199660:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:1.0:1787828580.199664:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1787828580.199670:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828580.199673:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828580.199682:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1787828580.199686:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828580.199690:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1787828580.199694:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1787828580.199698:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828580.199713:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:0.0:1787828580.199718:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787828580.199723:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787828580.199732:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00100000:1.0:1787828580.199732:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d13ec000 x1874670646744192/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787828580.199740:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787828580.199746:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828580.199756:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828580.199762:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828580.199767:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787828580.199775:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828580.199780:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1787828580.199786:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828580.199796:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1787828580.199799:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00100000:0.0:1787828580.199807:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d13ee940 x1874670646743936/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828605 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828580.199807:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787828580.199819:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1787828580.199831:0:468:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3380 00000100:00080000:1.0:1787828580.199845:0:468:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787828580.199846:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828580.199856:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787828580.199858:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787828580.199864:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:0.0:1787828580.199866:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:1.0:1787828580.199875:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:0.0:1787828580.199876:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828580.199881:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:1.0:1787828580.199883:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:0.0:1787828580.199886:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000020:00000001:1.0:1787828580.199891:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:0.0:1787828580.199893:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787828580.199906:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000020:00000001:1.0:1787828580.199906:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:1.0:1787828580.199917:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00080000:0.0:1787828580.199919:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787828580.199925:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:0.0:1787828580.199927:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787828580.199942:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d13ee940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646743936:192.168.203.143@tcp:250:kworker.0 00000100:00080000:1.0:1787828580.199945:0:468:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787828580.199951:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828580.199957:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828580.199959:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828580.199964:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787828580.199968:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787828580.199969:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787828580.199978:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 02000000:00000001:1.0:1787828580.199982:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000020:00000001:0.0:1787828580.199983:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787828580.199989:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787828580.199992:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828580.199999:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 00000100:00000001:1.0:1787828580.200001:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 02000000:00000001:0.0:1787828580.200004:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000008:00100000:1.0:1787828580.200009:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00000001:0.0:1787828580.200012:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787828580.200042:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000008:00000001:1.0:1787828580.200043:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828580.200051:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828580.200053:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828580.200065:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:0.0:1787828580.200067:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 02000000:00000001:1.0:1787828580.200069:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828580.200073:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828580.200077:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:0.0:1787828580.200079:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:1.0:1787828580.200086:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:0.0:1787828580.200087:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787828580.200094:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 02000000:00000001:1.0:1787828580.200095:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828580.200097:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787828580.200102:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828580.200107:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787828580.200112:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787828580.200112:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828580.200123:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828580.200135:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d13ec780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646744064:192.168.203.143@tcp:38:kworker.0 00000100:00100000:1.0:1787828580.200138:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d13ecb40 x1874670646744320/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787828580.200141:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787828580.200147:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787828580.200151:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828580.200159:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1787828580.200167:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828580.200174:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787828580.200181:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787828580.200184:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828580.200186:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839541340032 : -126234168211584 : ffff8d30d13ec780) 00000100:00000001:1.0:1787828580.200197:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828580.200198:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1787828580.200226:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:0.0:1787828580.200235:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1787828580.200241:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828580.200243:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828580.200246:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787828580.200252:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828580.200259:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298053143 00000100:00000001:0.0:1787828580.200263:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298054400 00000100:00000001:0.0:1787828580.200267:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298053143 00000100:00000001:0.0:1787828580.200270:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298054400 00000100:00000001:0.0:1787828580.200273:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787828580.200275:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828580.200277:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787828580.200283:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828580/real 1787828580] req@ffff8d30d13ec780 x1874670646744064/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828605 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787828580.200294:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787828580.200296:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787828580.200299:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787828580.200301:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787828580.200303:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828580.200305:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828580.200307:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787828580.200312:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d13ec780 x1874670646744064/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828605 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787828580.200320:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828580.200323:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787828580.200326:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828580.200328:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828580.200329:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828580.200332:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787828580.200334:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787828580.200338:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828580.200342:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787828580.200345:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787828580.200349:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d13ec780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646744064:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787828580.200354:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1787828580.200355:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787828580.200357:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828580.200359:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787828580.200362:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787828580.200364:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828580.200367:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d13ec000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646744192:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787828580.200371:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787828580.200373:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787828580.200375:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828580.200377:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1787828580.200380:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828580.200386:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787828580.200388:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839541338112 : -126234168213504 : ffff8d30d13ec000) 00000100:00000001:0.0:1787828580.200393:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1787828580.200404:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:0.0:1787828580.200409:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1787828580.200411:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828580.200413:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828580.200416:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787828580.200419:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828580.200422:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298053143 00000100:00000001:0.0:1787828580.200425:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298054400 00000100:00000001:0.0:1787828580.200428:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298053143 00000100:00000001:0.0:1787828580.200431:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298054400 00000100:00000001:0.0:1787828580.200434:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787828580.200436:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828580.200439:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787828580.200444:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828580/real 1787828580] req@ffff8d30d13ec000 x1874670646744192/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828605 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787828580.200454:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787828580.200455:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787828580.200458:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787828580.200459:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787828580.200461:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828580.200463:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828580.200465:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787828580.200469:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d13ec000 x1874670646744192/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828605 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787828580.200478:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828580.200481:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787828580.200483:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828580.200485:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828580.200487:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828580.200489:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787828580.200490:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787828580.200493:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828580.200497:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787828580.200499:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787828580.200503:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d13ec000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646744192:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787828580.200507:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828580.200509:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828580.200511:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787828580.200512:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787828580.200515:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787828580.200517:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787828580.200518:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787828580.200521:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787828580.200523:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787828580.200526:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787828580.200528:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828580.200530:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828580.200532:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787828580.200533:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787828580.200536:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787828580.200537:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787828580.200538:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787828580.200542:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787828580.200543:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787828580.200545:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787828580.200546:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828580.200548:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828580.200552:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787828580.200553:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828580.200556:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787828580.200557:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787828580.200559:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1787828580.200560:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787828580.200562:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828580.200565:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787828580.200567:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787828580.200569:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828580.200573:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d13ecb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646744320:192.168.203.143@tcp:8:kworker.0 00000100:00000001:0.0:1787828580.200577:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787828580.200579:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787828580.200581:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828580.200583:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1787828580.200586:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828580.200592:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787828580.200594:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839541340992 : -126234168210624 : ffff8d30d13ecb40) 00000100:00000001:0.0:1787828580.200636:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1787828580.200663:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:0.0:1787828580.200672:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1787828580.200676:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828580.200680:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828580.200683:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787828580.200688:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828580.200695:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298053157 00000100:00000001:0.0:1787828580.200702:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298054400 00000100:00000001:0.0:1787828580.200706:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298053157 00000100:00000001:0.0:1787828580.200711:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298054400 00000100:00000001:0.0:1787828580.200715:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787828580.200719:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828580.200722:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787828580.200729:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828580/real 1787828580] req@ffff8d30d13ecb40 x1874670646744320/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828605 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787828580.200739:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787828580.200741:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787828580.200746:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787828580.200750:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787828580.200753:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828580.200756:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828580.200759:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787828580.200766:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d13ecb40 x1874670646744320/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828605 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787828580.200780:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828580.200785:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787828580.200789:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828580.200793:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828580.200796:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828580.200799:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787828580.200803:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787828580.200809:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828580.200816:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787828580.200821:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787828580.200828:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d13ecb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646744320:192.168.203.143@tcp:8:kworker.0 00000100:00000001:0.0:1787828580.200835:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828580.200840:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828580.200842:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787828580.200844:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787828580.200847:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787828580.200849:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787828580.200851:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787828580.200854:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787828580.200855:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787828580.200858:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787828580.200860:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828580.200862:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828580.200867:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787828580.200869:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828580.200871:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787828580.200874:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828581.094745:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828581.094753:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787828581.094863:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787828581.094871:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787828581.094876:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828582.118742:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828582.118751:0:815:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1787828582.118754:0:815:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1787828582.118757:0:815:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828582.118762:0:815:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1787828582.118764:0:815:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828582.118766:0:815:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828582.118768:0:815:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828582.118837:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828582.118842:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828582.118846:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828583.142751:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828583.142759:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787828583.142853:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787828583.142862:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787828583.142868:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828584.166736:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828584.166743:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787828584.166826:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787828584.166831:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787828584.166835:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828585.190742:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828585.190749:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828585.190827:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828585.190833:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828585.190836:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787828585.318671:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828585.318681:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1787828585.318683:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1787828585.318686:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787828585.318690:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1787828585.318695:0:468:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 3385 00000100:00080000:1.0:1787828585.318704:0:468:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1787828585.318706:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787828585.318707:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:1.0:1787828585.318711:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:1.0:1787828585.318713:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:1.0:1787828585.318716:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:1.0:1787828585.318718:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:1.0:1787828585.318719:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787828585.318727:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:1.0:1787828585.318731:0:468:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:1.0:1787828585.318733:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828585.318735:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1787828585.318738:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787828585.318740:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828585.318742:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:1.0:1787828585.318744:0:468:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828585.318753:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1787828585.318754:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828585.318757:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1787828585.318759:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1787828585.318762:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828585.318771:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787828585.318778:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d13ec3c0 x1874670646744448/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787828585.318786:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787828585.318790:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828585.318801:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1787828585.318803:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828585.318806:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1787828585.318807:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1787828585.318808:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787828585.318810:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1787828585.318811:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3385 00000100:00080000:1.0:1787828585.318813:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1787828585.318815:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787828585.318816:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:1.0:1787828585.318817:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:1.0:1787828585.318819:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:1.0:1787828585.318821:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000020:00000001:1.0:1787828585.318822:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:1.0:1787828585.318824:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787828585.318825:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:1.0:1787828585.318827:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:1.0:1787828585.318828:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828585.318830:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1787828585.318834:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787828585.318835:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828585.318837:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1787828585.318841:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000008:00100000:1.0:1787828585.318841:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1787828585.318846:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828585.318847:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1787828585.318848:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828585.318852:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 02000000:00000001:1.0:1787828585.318853:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:0.0:1787828585.318856:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 02000000:00000001:1.0:1787828585.318856:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828585.318858:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:0.0:1787828585.318860:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828585.318861:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1787828585.318864:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828585.318866:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:1.0:1787828585.318869:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828585.318870:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787828585.318875:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787828585.318878:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d13ed680 x1874670646744576/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787828585.318882:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d13ec3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646744448:192.168.203.143@tcp:250:kworker.0 00000100:00000001:0.0:1787828585.318891:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:1.0:1787828585.318894:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 02000000:00000001:0.0:1787828585.318895:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:1.0:1787828585.318897:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828585.318899:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828585.318901:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828585.318903:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00080000:1.0:1787828585.318905:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828585.318908:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 02000000:00000001:0.0:1787828585.318909:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828585.318910:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1787828585.318914:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787828585.318916:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1787828585.318919:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3385 00000100:00080000:1.0:1787828585.318921:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1787828585.318923:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787828585.318924:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:0.0:1787828585.318926:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000020:00000001:1.0:1787828585.318926:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:1.0:1787828585.318927:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:0.0:1787828585.318929:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839541339072 : -126234168212544 : ffff8d30d13ec3c0) 00000020:00000001:1.0:1787828585.318931:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:0.0:1787828585.318935:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000020:00000001:1.0:1787828585.318935:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:1.0:1787828585.318939:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787828585.318941:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:1.0:1787828585.318943:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:1.0:1787828585.318945:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828585.318946:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1787828585.318948:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787828585.318949:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828585.318951:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:1.0:1787828585.318952:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1787828585.318953:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828585.318954:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828585.318958:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1787828585.318959:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828585.318960:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1787828585.318961:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1787828585.318962:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828585.318966:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:0.0:1787828585.318969:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00100000:1.0:1787828585.318974:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d13ed2c0 x1874670646744704/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787828585.318980:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787828585.318995:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828585.318997:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1787828585.318999:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828585.319001:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1787828585.319002:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1787828585.319003:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787828585.319005:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1787828585.319008:0:468:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3385 00000100:00080000:1.0:1787828585.319011:0:468:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1787828585.319012:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787828585.319013:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:1.0:1787828585.319014:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:1.0:1787828585.319015:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:1.0:1787828585.319016:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000020:00000001:1.0:1787828585.319018:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:1.0:1787828585.319019:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787828585.319020:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:1.0:1787828585.319021:0:468:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:1.0:1787828585.319023:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828585.319024:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1787828585.319026:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787828585.319027:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828585.319028:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:1.0:1787828585.319029:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000008:00000001:1.0:1787828585.319031:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828585.319032:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828585.319034:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1787828585.319035:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828585.319037:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1787828585.319037:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1787828585.319039:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828585.319043:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787828585.319046:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d13efc00 x1874670646744832/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787828585.319050:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787828585.319051:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828585.319053:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828585.319064:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1787828585.319072:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828585.319075:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828585.319078:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787828585.319085:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828585.319090:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4298059520 00000100:00000001:0.0:1787828585.319098:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1787828585.319103:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4298059520 00000100:00000001:0.0:1787828585.319106:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1787828585.319108:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787828585.319110:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828585.319112:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787828585.319117:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828585/real 1787828585] req@ffff8d30d13ec3c0 x1874670646744448/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828610 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787828585.319134:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787828585.319135:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787828585.319138:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787828585.319140:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787828585.319141:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828585.319143:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828585.319144:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787828585.319147:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d13ec3c0 x1874670646744448/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828610 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787828585.319153:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828585.319156:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787828585.319158:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828585.319159:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828585.319160:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828585.319161:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787828585.319162:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787828585.319164:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828585.319167:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787828585.319169:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787828585.319171:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d13ec3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646744448:192.168.203.143@tcp:250:kworker.0 00000100:00000001:0.0:1787828585.319175:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828585.319176:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828585.319178:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787828585.319180:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787828585.319182:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787828585.319184:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787828585.319185:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787828585.319188:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787828585.319189:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787828585.319192:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787828585.319192:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828585.319195:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828585.319201:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787828585.319202:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828585.319203:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787828585.319205:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787828585.319206:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1787828585.319207:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787828585.319208:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828585.319209:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787828585.319210:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787828585.319212:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828585.319214:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d13ed680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646744576:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787828585.319216:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787828585.319218:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787828585.319219:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828585.319220:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1787828585.319222:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828585.319226:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787828585.319227:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839541343872 : -126234168207744 : ffff8d30d13ed680) 00000100:00000001:0.0:1787828585.319230:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1787828585.319240:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:0.0:1787828585.319243:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1787828585.319244:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828585.319245:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828585.319246:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787828585.319248:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828585.319251:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298058262 00000100:00000001:0.0:1787828585.319254:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298059520 00000100:00000001:0.0:1787828585.319255:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298058262 00000100:00000001:0.0:1787828585.319257:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298059520 00000100:00000001:0.0:1787828585.319259:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787828585.319260:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828585.319261:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787828585.319264:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828585/real 1787828585] req@ffff8d30d13ed680 x1874670646744576/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828610 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787828585.319269:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787828585.319270:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787828585.319272:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787828585.319273:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787828585.319274:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828585.319275:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828585.319276:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787828585.319278:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d13ed680 x1874670646744576/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828610 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787828585.319283:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828585.319285:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787828585.319286:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828585.319287:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828585.319288:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828585.319289:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787828585.319290:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787828585.319291:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828585.319293:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787828585.319294:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787828585.319296:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d13ed680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646744576:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787828585.319299:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1787828585.319299:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787828585.319300:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828585.319302:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787828585.319303:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787828585.319305:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828585.319307:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d13ed2c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646744704:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787828585.319309:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787828585.319310:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787828585.319311:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828585.319312:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1787828585.319313:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828585.319316:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787828585.319317:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839541342912 : -126234168208704 : ffff8d30d13ed2c0) 00000100:00000001:0.0:1787828585.319319:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1787828585.319325:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:0.0:1787828585.319327:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1787828585.319328:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828585.319329:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828585.319330:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787828585.319332:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828585.319334:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298058262 00000100:00000001:0.0:1787828585.319336:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298059520 00000100:00000001:0.0:1787828585.319337:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298058262 00000100:00000001:0.0:1787828585.319339:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298059520 00000100:00000001:0.0:1787828585.319340:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787828585.319341:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828585.319342:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787828585.319345:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828585/real 1787828585] req@ffff8d30d13ed2c0 x1874670646744704/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828610 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787828585.319349:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787828585.319350:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787828585.319351:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787828585.319352:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787828585.319353:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828585.319354:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828585.319355:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787828585.319357:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d13ed2c0 x1874670646744704/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828610 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787828585.319362:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828585.319364:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787828585.319365:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828585.319366:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828585.319367:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828585.319368:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787828585.319369:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787828585.319370:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828585.319372:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787828585.319373:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787828585.319375:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d13ed2c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646744704:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787828585.319377:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1787828585.319378:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787828585.319379:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828585.319380:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787828585.319381:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787828585.319382:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828585.319384:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d13efc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646744832:192.168.203.143@tcp:8:kworker.0 00000100:00000001:0.0:1787828585.319386:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787828585.319387:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787828585.319388:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828585.319389:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1787828585.319390:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828585.319393:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787828585.319394:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839541353472 : -126234168198144 : ffff8d30d13efc00) 00000100:00000001:0.0:1787828585.319396:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1787828585.319402:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:0.0:1787828585.319404:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1787828585.319405:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828585.319407:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828585.319409:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787828585.319411:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828585.319414:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298058275 00000100:00000001:0.0:1787828585.319417:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298059520 00000100:00000001:0.0:1787828585.319419:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298058275 00000100:00000001:0.0:1787828585.319421:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298059520 00000100:00000001:0.0:1787828585.319423:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787828585.319425:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828585.319426:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787828585.319431:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828585/real 1787828585] req@ffff8d30d13efc00 x1874670646744832/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828610 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787828585.319439:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787828585.319441:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787828585.319444:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787828585.319445:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787828585.319447:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828585.319449:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828585.319450:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787828585.319454:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d13efc00 x1874670646744832/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828610 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787828585.319462:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828585.319464:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787828585.319466:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828585.319467:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828585.319468:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828585.319469:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787828585.319471:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787828585.319472:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828585.319474:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787828585.319475:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787828585.319478:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d13efc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646744832:192.168.203.143@tcp:8:kworker.0 00000100:00000001:0.0:1787828585.319480:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828585.319481:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828585.319482:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787828585.319484:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787828585.319485:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787828585.319486:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787828585.319487:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787828585.319488:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787828585.319489:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787828585.319491:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787828585.319492:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828585.319493:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828585.319494:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787828585.319495:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787828585.319496:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787828585.319497:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787828585.319497:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787828585.319499:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787828585.319500:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787828585.319500:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787828585.319501:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828585.319502:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828585.319503:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787828585.319503:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787828585.319505:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787828585.319505:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787828585.319506:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787828585.319507:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787828585.319507:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787828585.319509:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787828585.319509:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828585.319510:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828585.319513:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787828585.319513:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828585.319515:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787828585.319516:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828586.214736:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828586.214743:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787828586.214797:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787828586.214802:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787828586.214805:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828587.238772:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828587.238782:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787828587.238869:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787828587.238873:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787828587.238877:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828588.262690:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828588.262698:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828588.262922:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828588.262931:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828588.262938:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828589.286705:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828589.286712:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787828589.286770:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787828589.286775:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787828589.286795:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828590.310748:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828590.310755:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828590.310825:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828590.310830:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828590.310834:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787828590.438724:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828590.438733:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1787828590.438735:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1787828590.438738:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787828590.438741:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1787828590.438747:0:468:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 3390 00000100:00080000:1.0:1787828590.438756:0:468:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1787828590.438758:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787828590.438759:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:1.0:1787828590.438779:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:1.0:1787828590.438782:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:1.0:1787828590.438786:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:1.0:1787828590.438790:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:1.0:1787828590.438792:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787828590.438794:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:1.0:1787828590.438800:0:468:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:1.0:1787828590.438803:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828590.438807:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1787828590.438811:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787828590.438814:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828590.438817:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:1.0:1787828590.438820:0:468:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828590.438836:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1787828590.438837:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828590.438842:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1787828590.438846:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1787828590.438852:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828590.438865:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787828590.438877:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d13ef0c0 x1874670646744960/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787828590.438891:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787828590.438896:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828590.438917:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1787828590.438921:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828590.438926:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1787828590.438928:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1787828590.438930:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787828590.438933:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1787828590.438936:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3390 00000100:00080000:1.0:1787828590.438941:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1787828590.438943:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787828590.438983:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:1.0:1787828590.438985:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:1.0:1787828590.438987:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:1.0:1787828590.438988:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000020:00000001:1.0:1787828590.438990:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:1.0:1787828590.438992:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787828590.438993:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:1.0:1787828590.438996:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:1.0:1787828590.438998:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828590.439000:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:1.0:1787828590.439000:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1787828590.439004:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787828590.439005:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787828590.439009:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 02000000:00000001:1.0:1787828590.439009:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828590.439011:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1787828590.439013:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787828590.439016:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000008:00100000:1.0:1787828590.439016:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1787828590.439022:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828590.439023:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:1.0:1787828590.439025:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828590.439027:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787828590.439029:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787828590.439031:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828590.439048:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828590.439052:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1787828590.439055:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00100000:0.0:1787828590.439056:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d13ef0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646744960:192.168.203.143@tcp:250:kworker.0 02000000:00000001:1.0:1787828590.439057:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828590.439064:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:1.0:1787828590.439065:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828590.439069:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787828590.439073:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787828590.439074:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d13ede00 x1874670646745088/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:0.0:1787828590.439078:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:1.0:1787828590.439084:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 02000000:00000001:0.0:1787828590.439085:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828590.439088:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828590.439092:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1787828590.439095:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828590.439097:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:1.0:1787828590.439098:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1787828590.439100:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787828590.439102:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787828590.439102:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:0.0:1787828590.439106:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839541350592 : -126234168201024 : ffff8d30d13ef0c0) 00000100:00080000:1.0:1787828590.439107:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3390 00000100:00000001:0.0:1787828590.439113:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00080000:1.0:1787828590.439114:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1787828590.439118:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787828590.439119:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:1.0:1787828590.439121:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:1.0:1787828590.439122:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:1.0:1787828590.439123:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000020:00000001:1.0:1787828590.439125:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:1.0:1787828590.439127:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787828590.439128:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:1.0:1787828590.439131:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:1.0:1787828590.439133:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828590.439134:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1787828590.439136:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787828590.439138:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828590.439140:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:1.0:1787828590.439142:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1787828590.439144:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828590.439146:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828590.439150:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000800:00000100:0.0:1787828590.439151:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 02000000:00000001:1.0:1787828590.439152:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828590.439156:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1787828590.439157:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1787828590.439159:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828590.439163:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787828590.439163:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828590.439170:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00100000:1.0:1787828590.439172:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d13ef840 x1874670646745216/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787828590.439174:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828590.439179:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787828590.439184:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787828590.439186:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828590.439187:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828590.439194:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828590.439195:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4298064640 00000100:00080000:1.0:1787828590.439197:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828590.439202:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1787828590.439203:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1787828590.439204:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787828590.439206:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:0.0:1787828590.439207:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 00000100:00080000:1.0:1787828590.439209:0:468:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3390 00000100:00080000:1.0:1787828590.439214:0:468:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787828590.439216:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4298064640 00000100:00000001:1.0:1787828590.439216:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787828590.439217:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:1.0:1787828590.439223:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:0.0:1787828590.439224:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 00000020:00000001:1.0:1787828590.439225:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:1.0:1787828590.439228:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:0.0:1787828590.439230:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000020:00000001:1.0:1787828590.439231:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:0.0:1787828590.439237:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828590.439238:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787828590.439241:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:0.0:1787828590.439243:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00080000:1.0:1787828590.439246:0:468:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00100000:0.0:1787828590.439252:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828590/real 1787828590] req@ffff8d30d13ef0c0 x1874670646744960/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828615 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787828590.439252:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828590.439255:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1787828590.439259:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787828590.439263:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828590.439268:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:1.0:1787828590.439272:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000400:00000001:0.0:1787828590.439273:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000008:00000001:1.0:1787828590.439275:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:0.0:1787828590.439276:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787828590.439278:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828590.439281:0:2390:0:(events.c:78:reply_in_callback()) Process entered 02000000:00000001:1.0:1787828590.439283:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:0.0:1787828590.439285:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 02000000:00000001:1.0:1787828590.439287:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828590.439290:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828590.439290:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1787828590.439293:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:0.0:1787828590.439295:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 02000000:00000001:1.0:1787828590.439296:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828590.439299:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828590.439302:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828590.439305:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d13ef0c0 x1874670646744960/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828615 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828590.439310:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d13ef480 x1874670646745344/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787828590.439334:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828590.439335:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787828590.439340:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828590.439342:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787828590.439345:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828590.439346:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828590.439347:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828590.439349:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787828590.439351:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828590.439352:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787828590.439358:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828590.439363:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787828590.439365:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787828590.439368:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d13ef0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646744960:192.168.203.143@tcp:250:kworker.0 00000100:00000001:0.0:1787828590.439374:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828590.439376:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828590.439377:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787828590.439379:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787828590.439382:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787828590.439383:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787828590.439385:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787828590.439389:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787828590.439390:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787828590.439393:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787828590.439394:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828590.439396:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828590.439402:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787828590.439403:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828590.439405:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787828590.439406:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787828590.439408:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1787828590.439409:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787828590.439410:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828590.439412:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787828590.439413:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787828590.439415:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828590.439418:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d13ede00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646745088:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787828590.439420:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787828590.439422:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787828590.439423:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828590.439424:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1787828590.439426:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828590.439430:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787828590.439432:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839541345792 : -126234168205824 : ffff8d30d13ede00) 00000100:00000001:0.0:1787828590.439436:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1787828590.439444:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:0.0:1787828590.439448:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1787828590.439449:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828590.439450:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828590.439452:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787828590.439454:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828590.439457:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298063382 00000100:00000001:0.0:1787828590.439459:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298064640 00000100:00000001:0.0:1787828590.439461:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298063382 00000100:00000001:0.0:1787828590.439463:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298064640 00000100:00000001:0.0:1787828590.439465:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787828590.439466:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828590.439467:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787828590.439471:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828590/real 1787828590] req@ffff8d30d13ede00 x1874670646745088/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828615 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787828590.439477:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787828590.439478:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787828590.439479:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787828590.439497:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787828590.439498:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828590.439499:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828590.439500:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787828590.439503:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d13ede00 x1874670646745088/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828615 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787828590.439508:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828590.439509:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787828590.439511:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828590.439512:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828590.439512:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828590.439513:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787828590.439515:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787828590.439516:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828590.439518:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787828590.439519:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787828590.439521:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d13ede00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646745088:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787828590.439523:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1787828590.439524:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787828590.439525:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828590.439526:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787828590.439527:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787828590.439528:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828590.439530:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d13ef840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646745216:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787828590.439532:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787828590.439533:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787828590.439534:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828590.439535:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1787828590.439537:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828590.439540:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787828590.439540:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839541352512 : -126234168199104 : ffff8d30d13ef840) 00000100:00000001:0.0:1787828590.439543:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1787828590.439566:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:0.0:1787828590.439569:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1787828590.439570:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828590.439572:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828590.439573:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787828590.439575:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828590.439578:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298063382 00000100:00000001:0.0:1787828590.439579:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298064640 00000100:00000001:0.0:1787828590.439581:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298063382 00000100:00000001:0.0:1787828590.439583:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298064640 00000100:00000001:0.0:1787828590.439584:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787828590.439586:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828590.439587:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787828590.439590:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828590/real 1787828590] req@ffff8d30d13ef840 x1874670646745216/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828615 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787828590.439595:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787828590.439596:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787828590.439598:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787828590.439599:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787828590.439599:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828590.439601:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828590.439602:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787828590.439604:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d13ef840 x1874670646745216/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828615 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787828590.439609:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828590.439611:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787828590.439630:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828590.439632:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828590.439634:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828590.439636:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787828590.439638:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787828590.439640:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828590.439644:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787828590.439646:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787828590.439651:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d13ef840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646745216:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787828590.439655:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1787828590.439705:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787828590.439708:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828590.439711:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787828590.439714:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787828590.439716:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828590.439721:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d13ef480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646745344:192.168.203.143@tcp:8:kworker.0 00000100:00000001:0.0:1787828590.439740:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787828590.439742:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787828590.439743:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828590.439744:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1787828590.439746:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828590.439750:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787828590.439751:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839541351552 : -126234168200064 : ffff8d30d13ef480) 00000100:00000001:0.0:1787828590.439753:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1787828590.439761:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:0.0:1787828590.439764:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1787828590.439766:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828590.439767:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828590.439768:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787828590.439770:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828590.439772:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298063396 00000100:00000001:0.0:1787828590.439773:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298064640 00000100:00000001:0.0:1787828590.439775:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298063396 00000100:00000001:0.0:1787828590.439776:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298064640 00000100:00000001:0.0:1787828590.439777:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787828590.439779:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828590.439780:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787828590.439782:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828590/real 1787828590] req@ffff8d30d13ef480 x1874670646745344/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828615 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787828590.439810:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787828590.439812:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787828590.439815:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787828590.439817:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787828590.439818:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828590.439820:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828590.439822:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787828590.439826:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d13ef480 x1874670646745344/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828615 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787828590.439835:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828590.439838:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787828590.439841:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828590.439843:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828590.439844:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828590.439846:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787828590.439848:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787828590.439851:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828590.439855:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787828590.439857:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787828590.439861:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d13ef480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646745344:192.168.203.143@tcp:8:kworker.0 00000100:00000001:0.0:1787828590.439865:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828590.439867:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828590.439869:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787828590.439870:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787828590.439873:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787828590.439875:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787828590.439876:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787828590.439879:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787828590.439880:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787828590.439884:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787828590.439885:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828590.439888:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828590.439889:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787828590.439891:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787828590.439893:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787828590.439895:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787828590.439896:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787828590.439899:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787828590.439900:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787828590.439902:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787828590.439903:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828590.439905:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828590.439906:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787828590.439908:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787828590.439910:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787828590.439911:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787828590.439912:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787828590.439914:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787828590.439915:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787828590.439917:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787828590.439918:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828590.439920:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828590.439924:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787828590.439926:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828590.439928:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787828590.439930:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828591.334729:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828591.334735:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828591.334817:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828591.334821:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828591.334824:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828592.358717:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828592.358726:0:815:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1787828592.358729:0:815:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1787828592.358736:0:815:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828592.358741:0:815:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1787828592.358743:0:815:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828592.358744:0:815:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828592.358746:0:815:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787828592.358824:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787828592.358830:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787828592.358833:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828593.382727:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828593.382737:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828593.382812:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828593.382818:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828593.382821:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828594.406725:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828594.406731:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828594.406810:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828594.406815:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828594.406819:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828595.430673:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828595.430678:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787828595.430716:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787828595.430719:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787828595.430721:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787828595.558668:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828595.558679:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1787828595.558681:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1787828595.558685:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787828595.558689:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1787828595.558697:0:468:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 3396 00000100:00080000:1.0:1787828595.558706:0:468:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1787828595.558710:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787828595.558712:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:1.0:1787828595.558717:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:1.0:1787828595.558720:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:1.0:1787828595.558722:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:1.0:1787828595.558725:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:1.0:1787828595.558727:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787828595.558728:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:1.0:1787828595.558733:0:468:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:1.0:1787828595.558736:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828595.558738:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1787828595.558742:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787828595.558744:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828595.558747:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:1.0:1787828595.558750:0:468:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828595.558759:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1787828595.558761:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828595.558765:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1787828595.558767:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1787828595.558771:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828595.558781:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787828595.558790:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d13eed00 x1874670646745472/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787828595.558800:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787828595.558805:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828595.558819:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1787828595.558822:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828595.558825:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1787828595.558826:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1787828595.558828:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787828595.558831:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1787828595.558833:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3396 00000100:00080000:1.0:1787828595.558837:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1787828595.558840:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787828595.558841:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:1.0:1787828595.558843:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:1.0:1787828595.558845:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:1.0:1787828595.558847:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000020:00000001:1.0:1787828595.558849:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:1.0:1787828595.558851:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787828595.558853:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:1.0:1787828595.558856:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:1.0:1787828595.558858:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828595.558860:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1787828595.558865:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787828595.558868:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828595.558869:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:1.0:1787828595.558874:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1787828595.558877:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828595.558880:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828595.558884:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1787828595.558886:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828595.558888:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1787828595.558889:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1787828595.558891:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828595.558896:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828595.558898:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787828595.558904:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00100000:1.0:1787828595.558904:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d13ee580 x1874670646745600/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787828595.558907:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1787828595.558925:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787828595.558929:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828595.558935:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787828595.558938:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828595.558941:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:1.0:1787828595.558944:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828595.558945:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00080000:1.0:1787828595.558948:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 02000000:00000001:0.0:1787828595.558953:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1787828595.558955:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:1.0:1787828595.558959:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00100000:0.0:1787828595.558960:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d13eed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646745472:192.168.203.143@tcp:250:kworker.0 00000100:00080000:1.0:1787828595.558963:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787828595.558966:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:1.0:1787828595.558968:0:468:0:(import.c:485:import_select_connection()) Process entered 02000000:00000001:0.0:1787828595.558971:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787828595.558975:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1787828595.558975:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3396 02000000:00000001:0.0:1787828595.558978:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00080000:1.0:1787828595.558981:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 02000000:00000001:0.0:1787828595.558984:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828595.558985:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787828595.558989:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:1.0:1787828595.558992:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:1.0:1787828595.558993:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:1.0:1787828595.558995:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000020:00000001:1.0:1787828595.558997:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:0.0:1787828595.558998:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787828595.559000:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787828595.559001:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839541349632 : -126234168201984 : ffff8d30d13eed00) 00000100:00000001:1.0:1787828595.559004:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:0.0:1787828595.559006:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00080000:1.0:1787828595.559012:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:1.0:1787828595.559016:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828595.559018:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1787828595.559021:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787828595.559024:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828595.559025:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:1.0:1787828595.559027:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1787828595.559031:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828595.559033:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828595.559038:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000800:00000100:0.0:1787828595.559041:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 02000000:00000001:1.0:1787828595.559041:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828595.559046:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1787828595.559050:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1787828595.559052:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828595.559054:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1787828595.559058:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787828595.559059:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828595.559061:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828595.559064:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787828595.559069:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787828595.559070:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d13ee1c0 x1874670646745728/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787828595.559073:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4298069760 00000100:00000001:0.0:1787828595.559082:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 00000100:00000001:1.0:1787828595.559083:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787828595.559087:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828595.559089:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4298069760 00000100:00000001:1.0:1787828595.559091:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828595.559094:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 00000100:00080000:1.0:1787828595.559096:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1787828595.559098:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787828595.559103:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828595.559106:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00080000:1.0:1787828595.559106:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1787828595.559111:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00100000:0.0:1787828595.559114:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828595/real 1787828595] req@ffff8d30d13eed00 x1874670646745472/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828620 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828595.559115:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787828595.559122:0:468:0:(import.c:485:import_select_connection()) Process entered 00000400:00000001:0.0:1787828595.559128:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00080000:1.0:1787828595.559128:0:468:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3396 00000400:00000001:0.0:1787828595.559130:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787828595.559133:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787828595.559136:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00080000:1.0:1787828595.559136:0:468:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787828595.559139:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828595.559141:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787828595.559142:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787828595.559145:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:0.0:1787828595.559146:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:1.0:1787828595.559151:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000100:00100000:0.0:1787828595.559152:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d13eed00 x1874670646745472/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828620 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:1.0:1787828595.559155:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:1.0:1787828595.559160:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:0.0:1787828595.559166:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:1.0:1787828595.559166:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:1.0:1787828595.559169:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787828595.559171:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787828595.559172:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:0.0:1787828595.559179:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:1.0:1787828595.559181:0:468:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787828595.559182:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828595.559184:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828595.559186:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828595.559187:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 02000000:00000001:1.0:1787828595.559189:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:0.0:1787828595.559190:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 02000000:00000001:1.0:1787828595.559193:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00080000:0.0:1787828595.559195:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 02000000:00000001:1.0:1787828595.559198:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787828595.559201:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787828595.559201:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:1.0:1787828595.559204:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00000001:0.0:1787828595.559205:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000008:00000001:1.0:1787828595.559209:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828595.559211:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d13eed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646745472:192.168.203.143@tcp:250:kworker.0 00000100:00000001:1.0:1787828595.559213:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828595.559216:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828595.559219:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 02000000:00000001:1.0:1787828595.559220:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:0.0:1787828595.559221:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787828595.559225:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787828595.559225:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828595.559228:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00000001:1.0:1787828595.559230:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000020:00000001:0.0:1787828595.559231:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787828595.559234:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787828595.559234:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1787828595.559239:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828595.559240:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787828595.559244:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787828595.559247:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787828595.559248:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828595.559249:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828595.559250:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828595.559259:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00100000:1.0:1787828595.559261:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d13eda40 x1874670646745856/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787828595.559262:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828595.559266:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787828595.559268:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787828595.559270:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1787828595.559273:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787828595.559275:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828595.559278:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787828595.559280:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787828595.559284:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828595.559291:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d13ee580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646745600:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787828595.559297:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787828595.559298:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787828595.559301:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787828595.559304:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828595.559304:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828595.559305:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1787828595.559309:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828595.559312:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828595.559313:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787828595.559316:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839541347712 : -126234168203904 : ffff8d30d13ee580) 00000100:00000001:0.0:1787828595.559320:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1787828595.559330:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:0.0:1787828595.559333:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1787828595.559335:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828595.559336:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828595.559337:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787828595.559339:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828595.559342:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298068502 00000100:00000001:0.0:1787828595.559344:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298069760 00000100:00000001:0.0:1787828595.559346:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298068502 00000100:00000001:0.0:1787828595.559347:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298069760 00000100:00000001:0.0:1787828595.559348:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787828595.559350:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828595.559351:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787828595.559353:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828595/real 1787828595] req@ffff8d30d13ee580 x1874670646745600/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828620 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787828595.559359:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787828595.559359:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787828595.559360:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787828595.559362:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787828595.559363:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828595.559364:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828595.559365:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787828595.559367:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d13ee580 x1874670646745600/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828620 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787828595.559371:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828595.559373:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787828595.559374:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828595.559375:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828595.559376:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828595.559377:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787828595.559378:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787828595.559379:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828595.559382:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787828595.559383:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787828595.559385:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d13ee580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646745600:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787828595.559389:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1787828595.559390:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787828595.559391:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828595.559393:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787828595.559394:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787828595.559397:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828595.559400:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d13ee1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646745728:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787828595.559403:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787828595.559405:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787828595.559406:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828595.559408:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1787828595.559411:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828595.559416:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787828595.559418:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839541346752 : -126234168204864 : ffff8d30d13ee1c0) 00000100:00000001:0.0:1787828595.559421:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1787828595.559433:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:0.0:1787828595.559438:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1787828595.559440:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828595.559441:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828595.559443:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787828595.559446:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828595.559449:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298068502 00000100:00000001:0.0:1787828595.559452:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298069760 00000100:00000001:0.0:1787828595.559454:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298068502 00000100:00000001:0.0:1787828595.559456:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298069760 00000100:00000001:0.0:1787828595.559457:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787828595.559458:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828595.559460:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787828595.559463:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828595/real 1787828595] req@ffff8d30d13ee1c0 x1874670646745728/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828620 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787828595.559469:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787828595.559470:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787828595.559471:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787828595.559472:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787828595.559473:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828595.559475:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828595.559476:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787828595.559478:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d13ee1c0 x1874670646745728/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828620 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787828595.559482:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828595.559484:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787828595.559485:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828595.559486:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828595.559487:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828595.559488:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787828595.559489:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787828595.559490:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828595.559493:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787828595.559494:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787828595.559496:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d13ee1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646745728:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787828595.559498:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828595.559500:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828595.559500:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787828595.559501:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787828595.559503:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787828595.559504:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787828595.559504:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787828595.559506:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787828595.559507:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787828595.559508:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787828595.559509:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828595.559510:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828595.559511:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787828595.559512:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787828595.559513:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787828595.559514:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787828595.559515:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787828595.559516:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787828595.559516:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787828595.559517:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787828595.559518:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828595.559519:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828595.559521:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787828595.559522:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828595.559523:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787828595.559524:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787828595.559525:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1787828595.559526:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787828595.559527:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828595.559528:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787828595.559529:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787828595.559530:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828595.559532:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d13eda40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646745856:192.168.203.143@tcp:8:kworker.0 00000100:00000001:0.0:1787828595.559534:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787828595.559536:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787828595.559537:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828595.559538:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1787828595.559540:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828595.559543:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787828595.559544:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839541344832 : -126234168206784 : ffff8d30d13eda40) 00000100:00000001:0.0:1787828595.559547:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1787828595.559554:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:0.0:1787828595.559557:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1787828595.559558:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828595.559559:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828595.559560:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787828595.559562:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828595.559564:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298068515 00000100:00000001:0.0:1787828595.559566:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298069760 00000100:00000001:0.0:1787828595.559568:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298068515 00000100:00000001:0.0:1787828595.559569:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298069760 00000100:00000001:0.0:1787828595.559570:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787828595.559571:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828595.559573:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787828595.559575:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828595/real 1787828595] req@ffff8d30d13eda40 x1874670646745856/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828620 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787828595.559579:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787828595.559580:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787828595.559581:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787828595.559582:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787828595.559583:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828595.559585:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828595.559585:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787828595.559588:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d13eda40 x1874670646745856/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828620 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787828595.559592:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828595.559593:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787828595.559595:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828595.559596:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828595.559596:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828595.559597:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787828595.559628:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787828595.559630:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828595.559632:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787828595.559634:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787828595.559636:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d13eda40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646745856:192.168.203.143@tcp:8:kworker.0 00000100:00000001:0.0:1787828595.559639:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828595.559640:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828595.559641:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787828595.559642:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787828595.559644:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787828595.559645:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787828595.559646:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787828595.559647:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787828595.559648:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787828595.559650:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787828595.559650:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828595.559652:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828595.559653:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787828595.559654:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828595.559656:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787828595.559657:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828596.454736:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828596.454744:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828596.454798:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828596.454802:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828596.454805:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828597.478678:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828597.478684:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828597.478764:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828597.478768:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828597.478770:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828598.502727:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828598.502734:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787828598.502799:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787828598.502803:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787828598.502807:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828599.526679:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828599.526684:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828599.526748:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828599.526752:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828599.526756:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828600.550694:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828600.550699:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828600.550748:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828600.550752:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828600.550754:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787828600.678673:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828600.678683:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1787828600.678685:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1787828600.678689:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787828600.678694:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1787828600.678701:0:468:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 3401 00000100:00080000:1.0:1787828600.678712:0:468:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1787828600.678715:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787828600.678717:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:1.0:1787828600.678722:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:1.0:1787828600.678724:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:1.0:1787828600.678728:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:1.0:1787828600.678730:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:1.0:1787828600.678732:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787828600.678734:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:1.0:1787828600.678739:0:468:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:1.0:1787828600.678742:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828600.678745:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1787828600.678748:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787828600.678751:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828600.678754:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:1.0:1787828600.678757:0:468:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828600.678766:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1787828600.678768:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828600.678772:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1787828600.678775:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1787828600.678779:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828600.678789:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787828600.678797:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d13ecf00 x1874670646745984/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787828600.678808:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787828600.678812:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828600.678827:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1787828600.678831:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828600.678835:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1787828600.678836:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1787828600.678838:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787828600.678841:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1787828600.678844:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3401 00000100:00080000:1.0:1787828600.678847:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1787828600.678850:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787828600.678851:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:1.0:1787828600.678854:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:1.0:1787828600.678855:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:1.0:1787828600.678857:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000020:00000001:1.0:1787828600.678859:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:1.0:1787828600.678861:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787828600.678863:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:1.0:1787828600.678867:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:1.0:1787828600.678870:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828600.678872:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1787828600.678889:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787828600.678893:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828600.678895:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:1.0:1787828600.678900:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1787828600.678903:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828600.678906:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828600.678911:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:0.0:1787828600.678914:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:1.0:1787828600.678914:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828600.678920:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1787828600.678923:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:0.0:1787828600.678925:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 02000000:00000001:1.0:1787828600.678926:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828600.678932:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787828600.678936:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828600.678937:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787828600.678943:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787828600.678949:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d13eda40 x1874670646746112/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:0.0:1787828600.678953:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787828600.678959:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787828600.678968:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828600.678970:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787828600.678975:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828600.678982:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d13ecf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646745984:192.168.203.143@tcp:250:kworker.0 00000100:00000001:1.0:1787828600.678983:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1787828600.678991:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1787828600.678994:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00080000:1.0:1787828600.678999:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 02000000:00000001:0.0:1787828600.679004:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:1.0:1787828600.679004:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1787828600.679006:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 02000000:00000001:0.0:1787828600.679009:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828600.679012:0:468:0:(import.c:485:import_select_connection()) Process entered 02000000:00000001:0.0:1787828600.679020:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00080000:1.0:1787828600.679027:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3401 02000000:00000001:0.0:1787828600.679031:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1787828600.679034:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1787828600.679039:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787828600.679041:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:1.0:1787828600.679045:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:1.0:1787828600.679046:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:1.0:1787828600.679048:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000020:00000001:1.0:1787828600.679051:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:1.0:1787828600.679054:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787828600.679057:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787828600.679057:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:0.0:1787828600.679062:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839541341952 : -126234168209664 : ffff8d30d13ecf00) 00000100:00080000:1.0:1787828600.679067:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787828600.679072:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:1.0:1787828600.679072:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828600.679078:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1787828600.679082:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787828600.679085:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828600.679088:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:1.0:1787828600.679091:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1787828600.679094:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828600.679096:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828600.679100:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1787828600.679102:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828600.679104:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1787828600.679105:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1787828600.679108:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828600.679114:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787828600.679120:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d13ee1c0 x1874670646746240/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787828600.679126:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787828600.679141:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787828600.679144:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828600.679148:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828600.679150:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00080000:1.0:1787828600.679152:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828600.679157:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1787828600.679158:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787828600.679160:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:0.0:1787828600.679164:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1787828600.679165:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787828600.679172:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787828600.679174:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1787828600.679180:0:468:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3401 00000100:00000001:0.0:1787828600.679182:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1787828600.679186:0:468:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787828600.679190:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4298074880 00000100:00000001:1.0:1787828600.679191:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787828600.679193:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:1.0:1787828600.679202:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:1.0:1787828600.679203:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:1.0:1787828600.679205:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000020:00000001:1.0:1787828600.679208:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:0.0:1787828600.679210:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 00000100:00000001:1.0:1787828600.679212:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787828600.679216:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:0.0:1787828600.679222:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4298074880 00000100:00080000:1.0:1787828600.679223:0:468:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:1.0:1787828600.679232:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828600.679233:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 02000000:00000001:1.0:1787828600.679237:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:0.0:1787828600.679241:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787828600.679242:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787828600.679250:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828600.679250:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828600.679262:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1787828600.679264:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000008:00100000:1.0:1787828600.679267:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000008:00000001:1.0:1787828600.679272:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828600.679277:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828600/real 1787828600] req@ffff8d30d13ecf00 x1874670646745984/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828625 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787828600.679277:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828600.679288:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1787828600.679294:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828600.679303:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1787828600.679310:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000400:00000001:0.0:1787828600.679313:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 02000000:00000001:1.0:1787828600.679314:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:0.0:1787828600.679318:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787828600.679323:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828600.679325:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787828600.679331:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787828600.679334:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787828600.679334:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d13ee580 x1874670646746368/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787828600.679343:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828600.679348:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828600.679359:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00100000:0.0:1787828600.679366:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d13ecf00 x1874670646745984/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828625 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787828600.679367:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828600.679374:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828600.679387:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828600.679392:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787828600.679396:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828600.679400:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828600.679402:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828600.679406:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787828600.679409:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787828600.679412:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828600.679419:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787828600.679422:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787828600.679426:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d13ecf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646745984:192.168.203.143@tcp:250:kworker.0 00000100:00000001:0.0:1787828600.679433:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828600.679435:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828600.679437:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787828600.679440:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787828600.679444:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787828600.679447:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787828600.679449:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787828600.679454:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787828600.679456:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787828600.679461:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787828600.679462:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828600.679465:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828600.679473:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787828600.679476:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828600.679478:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787828600.679480:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787828600.679482:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1787828600.679483:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787828600.679485:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828600.679488:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787828600.679490:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787828600.679493:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828600.679496:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d13eda40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646746112:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787828600.679500:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787828600.679502:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787828600.679504:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828600.679506:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1787828600.679508:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828600.679515:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787828600.679516:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839541344832 : -126234168206784 : ffff8d30d13eda40) 00000100:00000001:0.0:1787828600.679521:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1787828600.679534:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:0.0:1787828600.679538:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1787828600.679541:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828600.679542:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828600.679544:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787828600.679547:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828600.679552:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298073622 00000100:00000001:0.0:1787828600.679556:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298074880 00000100:00000001:0.0:1787828600.679558:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298073622 00000100:00000001:0.0:1787828600.679561:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298074880 00000100:00000001:0.0:1787828600.679564:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787828600.679566:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828600.679567:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787828600.679572:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828600/real 1787828600] req@ffff8d30d13eda40 x1874670646746112/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828625 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787828600.679582:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787828600.679583:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787828600.679585:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787828600.679586:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787828600.679588:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828600.679590:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828600.679591:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787828600.679596:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d13eda40 x1874670646746112/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828625 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787828600.679647:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828600.679649:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787828600.679652:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828600.679653:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828600.679655:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828600.679657:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787828600.679659:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787828600.679662:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828600.679665:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787828600.679667:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787828600.679671:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d13eda40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646746112:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787828600.679675:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1787828600.679677:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787828600.679679:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828600.679681:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787828600.679683:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787828600.679685:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828600.679688:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d13ee1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646746240:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787828600.679692:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787828600.679694:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787828600.679696:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828600.679698:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1787828600.679701:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828600.679706:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787828600.679707:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839541346752 : -126234168204864 : ffff8d30d13ee1c0) 00000100:00000001:0.0:1787828600.679711:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1787828600.679722:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:0.0:1787828600.679726:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1787828600.679729:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828600.679730:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828600.679732:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787828600.679734:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828600.679738:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298073623 00000100:00000001:0.0:1787828600.679740:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298074880 00000100:00000001:0.0:1787828600.679743:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298073623 00000100:00000001:0.0:1787828600.679745:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298074880 00000100:00000001:0.0:1787828600.679748:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787828600.679750:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828600.679751:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787828600.679755:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828600/real 1787828600] req@ffff8d30d13ee1c0 x1874670646746240/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828625 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787828600.679764:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787828600.679765:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787828600.679768:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787828600.679769:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787828600.679771:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828600.679773:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828600.679774:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787828600.679778:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d13ee1c0 x1874670646746240/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828625 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787828600.679787:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828600.679789:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787828600.679791:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828600.679793:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828600.679794:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828600.679796:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787828600.679798:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787828600.679800:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828600.679803:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787828600.679806:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787828600.679809:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d13ee1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646746240:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787828600.679813:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1787828600.679815:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787828600.679816:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828600.679819:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787828600.679820:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787828600.679822:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828600.679825:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d13ee580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646746368:192.168.203.143@tcp:8:kworker.0 00000100:00000001:0.0:1787828600.679829:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787828600.679831:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787828600.679833:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828600.679835:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1787828600.679837:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828600.679842:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787828600.679844:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839541347712 : -126234168203904 : ffff8d30d13ee580) 00000100:00000001:0.0:1787828600.679847:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1787828600.679856:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:0.0:1787828600.679860:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1787828600.679862:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828600.679864:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828600.679866:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787828600.679868:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828600.679872:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298073636 00000100:00000001:0.0:1787828600.679891:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298074880 00000100:00000001:0.0:1787828600.679894:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298073636 00000100:00000001:0.0:1787828600.679896:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298074880 00000100:00000001:0.0:1787828600.679899:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787828600.679900:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828600.679902:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787828600.679906:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828600/real 1787828600] req@ffff8d30d13ee580 x1874670646746368/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828625 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787828600.679916:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787828600.679917:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787828600.679920:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787828600.679921:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787828600.679923:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828600.679925:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828600.679926:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787828600.679930:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d13ee580 x1874670646746368/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828625 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787828600.679938:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828600.679940:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787828600.679943:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828600.679944:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828600.679946:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828600.679948:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787828600.679950:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787828600.679952:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828600.679955:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787828600.679958:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787828600.679962:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d13ee580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646746368:192.168.203.143@tcp:8:kworker.0 00000100:00000001:0.0:1787828600.679966:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828600.679968:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828600.679970:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787828600.679971:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787828600.679973:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787828600.679975:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787828600.679976:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787828600.679979:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787828600.679981:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787828600.679983:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787828600.679984:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828600.679986:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828600.679988:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787828600.679989:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787828600.679991:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787828600.679992:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787828600.679993:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787828600.679994:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787828600.679996:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787828600.679998:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787828600.679999:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828600.680001:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828600.680002:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787828600.680004:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787828600.680005:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787828600.680007:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787828600.680008:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787828600.680010:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787828600.680011:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787828600.680012:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787828600.680014:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828600.680016:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828600.680019:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787828600.680021:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828600.680023:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787828600.680025:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828601.574662:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828601.574671:0:815:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1787828601.574674:0:815:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1787828601.574677:0:815:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828601.574683:0:815:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1787828601.574685:0:815:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828601.574688:0:815:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828601.574691:0:815:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787828601.574741:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787828601.574746:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787828601.574749:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828602.598669:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828602.598674:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828602.598719:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828602.598724:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828602.598727:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828603.622684:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828603.622692:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828603.622747:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828603.622752:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828603.622755:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828604.646651:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828604.646656:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787828604.646710:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787828604.646713:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787828604.646716:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828605.670690:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828605.670697:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828605.670739:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828605.670743:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828605.670745:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787828605.798718:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828605.798727:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1787828605.798729:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787828605.798733:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787828605.798738:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787828605.798745:0:468:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 3406 00000100:00080000:2.0:1787828605.798756:0:468:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787828605.798760:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828605.798762:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787828605.798766:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787828605.798769:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787828605.798772:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:2.0:1787828605.798775:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:2.0:1787828605.798778:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828605.798779:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:2.0:1787828605.798784:0:468:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787828605.798787:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828605.798790:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787828605.798794:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828605.798798:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828605.798800:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:2.0:1787828605.798803:0:468:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828605.798815:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787828605.798816:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828605.798818:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787828605.798820:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787828605.798823:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828605.798833:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787828605.798856:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2fc00 x1874670646746496/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787828605.798865:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787828605.798868:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828605.798880:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787828605.798884:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828605.798886:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1787828605.798887:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787828605.798888:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787828605.798891:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787828605.798892:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3406 00000100:00080000:2.0:1787828605.798895:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787828605.798896:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828605.798897:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787828605.798899:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787828605.798900:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787828605.798902:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000020:00000001:2.0:1787828605.798905:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:2.0:1787828605.798908:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828605.798909:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:2.0:1787828605.798912:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787828605.798914:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828605.798916:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:0.0:1787828605.798917:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:2.0:1787828605.798921:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787828605.798922:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 02000000:00000001:2.0:1787828605.798926:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828605.798927:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1787828605.798929:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1787828605.798930:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787828605.798933:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000008:00100000:2.0:1787828605.798935:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 02000000:00000001:0.0:1787828605.798937:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000008:00000001:2.0:1787828605.798939:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828605.798941:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787828605.798943:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828605.798945:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828605.798948:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787828605.798952:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828605.798953:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646746496:192.168.203.143@tcp:250:kworker.0 00000100:00000001:2.0:1787828605.798956:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787828605.798960:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:0.0:1787828605.798961:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787828605.798964:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828605.798965:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787828605.798969:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828605.798971:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:2.0:1787828605.798971:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828605.798978:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787828605.798979:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2de00 x1874670646746624/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787828605.798986:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787828605.798989:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828605.798992:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787828605.798993:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828605.798996:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417883648 : -126234291667968 : ffff8d30c9e2fc00) 00000100:00080000:2.0:1787828605.798997:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1787828605.799002:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00080000:2.0:1787828605.799003:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1787828605.799006:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787828605.799007:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787828605.799010:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787828605.799012:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3406 00000100:00080000:2.0:1787828605.799016:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787828605.799017:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828605.799019:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787828605.799021:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787828605.799022:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787828605.799024:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000020:00000001:2.0:1787828605.799026:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:2.0:1787828605.799028:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828605.799029:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:2.0:1787828605.799031:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787828605.799034:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:0.0:1787828605.799038:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 02000000:00000001:2.0:1787828605.799038:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787828605.799041:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828605.799044:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828605.799047:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1787828605.799049:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000008:00100000:2.0:1787828605.799050:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1787828605.799054:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828605.799056:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828605.799056:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828605.799061:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828605.799062:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787828605.799064:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828605.799067:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1787828605.799068:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787828605.799072:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787828605.799074:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828605.799077:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828605.799080:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828605.799083:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4298080000 00000100:00100000:2.0:1787828605.799089:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2c780 x1874670646746752/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787828605.799097:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 00000100:00000001:2.0:1787828605.799101:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787828605.799105:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828605.799106:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4298080000 00000100:00000001:2.0:1787828605.799109:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828605.799114:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 00000100:00080000:2.0:1787828605.799114:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1787828605.799121:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00080000:2.0:1787828605.799122:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1787828605.799126:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828605.799127:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:0.0:1787828605.799129:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00080000:2.0:1787828605.799132:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1787828605.799137:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828605/real 1787828605] req@ffff8d30c9e2fc00 x1874670646746496/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828630 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787828605.799139:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787828605.799147:0:468:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3406 00000400:00000001:0.0:1787828605.799152:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00080000:2.0:1787828605.799154:0:468:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000400:00000001:0.0:1787828605.799155:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1787828605.799160:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787828605.799161:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1787828605.799163:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:0.0:1787828605.799165:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787828605.799169:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:2.0:1787828605.799169:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:0.0:1787828605.799175:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000020:00000001:2.0:1787828605.799175:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:0.0:1787828605.799178:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:2.0:1787828605.799180:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00100000:0.0:1787828605.799184:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2fc00 x1874670646746496/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828630 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:2.0:1787828605.799186:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:2.0:1787828605.799192:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828605.799195:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:0.0:1787828605.799203:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:2.0:1787828605.799205:0:468:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787828605.799209:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787828605.799211:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828605.799214:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:2.0:1787828605.799216:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:0.0:1787828605.799220:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 02000000:00000001:2.0:1787828605.799222:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787828605.799223:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828605.799230:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 02000000:00000001:2.0:1787828605.799230:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828605.799234:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787828605.799235:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00080000:0.0:1787828605.799241:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000008:00100000:2.0:1787828605.799242:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00080000:0.0:1787828605.799247:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000008:00000001:2.0:1787828605.799248:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828605.799252:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000001:2.0:1787828605.799253:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828605.799259:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646746496:192.168.203.143@tcp:250:kworker.0 02000000:00000001:2.0:1787828605.799261:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:0.0:1787828605.799264:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:2.0:1787828605.799265:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828605.799268:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828605.799269:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:0.0:1787828605.799271:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787828605.799273:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787828605.799274:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1787828605.799277:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828605.799279:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787828605.799283:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787828605.799285:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 00000100:00000001:2.0:1787828605.799287:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828605.799289:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787828605.799292:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787828605.799294:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00100000:2.0:1787828605.799296:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2e940 x1874670646746880/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787828605.799297:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828605.799301:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828605.799309:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787828605.799310:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1787828605.799312:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828605.799313:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828605.799316:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828605.799317:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787828605.799321:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787828605.799323:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1787828605.799324:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787828605.799325:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828605.799327:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787828605.799329:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787828605.799331:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828605.799334:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646746624:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787828605.799337:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787828605.799338:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787828605.799339:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828605.799341:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1787828605.799343:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828605.799348:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787828605.799350:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417875968 : -126234291675648 : ffff8d30c9e2de00) 00000100:00000001:0.0:1787828605.799354:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1787828605.799363:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:0.0:1787828605.799366:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1787828605.799368:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828605.799369:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828605.799370:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787828605.799372:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828605.799375:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298078742 00000100:00000001:0.0:1787828605.799377:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298080000 00000100:00000001:0.0:1787828605.799379:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298078742 00000100:00000001:0.0:1787828605.799380:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298080000 00000100:00000001:0.0:1787828605.799382:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787828605.799383:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828605.799384:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787828605.799387:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828605/real 1787828605] req@ffff8d30c9e2de00 x1874670646746624/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828630 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787828605.799392:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787828605.799393:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787828605.799394:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787828605.799395:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787828605.799396:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828605.799397:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828605.799398:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787828605.799401:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2de00 x1874670646746624/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828630 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787828605.799405:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828605.799407:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787828605.799408:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828605.799409:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828605.799410:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828605.799411:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787828605.799412:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787828605.799414:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828605.799415:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787828605.799417:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787828605.799419:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646746624:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787828605.799421:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1787828605.799422:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787828605.799422:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828605.799424:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787828605.799425:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787828605.799426:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828605.799427:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646746752:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787828605.799429:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787828605.799430:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787828605.799431:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828605.799432:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1787828605.799434:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828605.799437:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787828605.799438:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417870208 : -126234291681408 : ffff8d30c9e2c780) 00000100:00000001:0.0:1787828605.799440:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1787828605.799445:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:0.0:1787828605.799448:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1787828605.799449:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828605.799450:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828605.799451:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787828605.799453:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828605.799455:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298078742 00000100:00000001:0.0:1787828605.799457:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298080000 00000100:00000001:0.0:1787828605.799458:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298078742 00000100:00000001:0.0:1787828605.799460:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298080000 00000100:00000001:0.0:1787828605.799461:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787828605.799462:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828605.799463:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787828605.799466:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828605/real 1787828605] req@ffff8d30c9e2c780 x1874670646746752/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828630 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787828605.799470:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787828605.799471:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787828605.799472:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787828605.799473:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787828605.799474:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828605.799474:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828605.799475:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787828605.799477:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2c780 x1874670646746752/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828630 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787828605.799482:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828605.799483:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787828605.799485:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828605.799486:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828605.799487:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828605.799488:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787828605.799489:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787828605.799490:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828605.799492:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787828605.799493:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787828605.799495:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646746752:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787828605.799498:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1787828605.799498:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787828605.799499:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828605.799501:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787828605.799502:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787828605.799503:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828605.799504:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646746880:192.168.203.143@tcp:8:kworker.0 00000100:00000001:0.0:1787828605.799506:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787828605.799507:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787828605.799508:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828605.799509:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1787828605.799510:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828605.799513:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787828605.799514:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417878848 : -126234291672768 : ffff8d30c9e2e940) 00000100:00000001:0.0:1787828605.799516:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1787828605.799521:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:0.0:1787828605.799523:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1787828605.799524:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828605.799525:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828605.799526:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787828605.799527:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828605.799529:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298078755 00000100:00000001:0.0:1787828605.799531:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298080000 00000100:00000001:0.0:1787828605.799532:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298078755 00000100:00000001:0.0:1787828605.799534:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298080000 00000100:00000001:0.0:1787828605.799535:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787828605.799536:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828605.799537:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787828605.799539:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828605/real 1787828605] req@ffff8d30c9e2e940 x1874670646746880/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828630 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787828605.799549:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787828605.799549:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787828605.799551:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787828605.799552:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787828605.799552:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828605.799553:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828605.799554:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787828605.799556:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2e940 x1874670646746880/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828630 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787828605.799560:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828605.799562:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787828605.799564:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828605.799565:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828605.799567:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828605.799568:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787828605.799570:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787828605.799572:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828605.799575:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787828605.799577:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787828605.799580:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646746880:192.168.203.143@tcp:8:kworker.0 00000100:00000001:0.0:1787828605.799584:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828605.799586:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828605.799588:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787828605.799589:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787828605.799590:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787828605.799591:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787828605.799592:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787828605.799594:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787828605.799595:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787828605.799622:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787828605.799624:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828605.799625:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828605.799626:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787828605.799627:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787828605.799628:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787828605.799629:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787828605.799630:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787828605.799631:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787828605.799632:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787828605.799634:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787828605.799635:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828605.799636:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828605.799636:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787828605.799637:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787828605.799638:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787828605.799639:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787828605.799639:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787828605.799640:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787828605.799641:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787828605.799642:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787828605.799643:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828605.799644:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828605.799646:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787828605.799647:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828605.799648:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787828605.799649:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828606.694666:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828606.694672:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828606.694713:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828606.694717:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828606.694720:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828607.718682:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828607.718688:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828607.718739:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828607.718744:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828607.718747:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828608.742684:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828608.742689:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828608.742738:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828608.742741:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828608.742744:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828609.766664:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828609.766670:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828609.766725:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828609.766729:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828609.766732:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828610.790707:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828610.790712:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828610.790768:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828610.790772:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828610.790776:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787828610.918666:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828610.918674:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1787828610.918677:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787828610.918680:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787828610.918684:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787828610.918690:0:468:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 3411 00000100:00080000:2.0:1787828610.918698:0:468:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787828610.918700:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828610.918702:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787828610.918705:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787828610.918708:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787828610.918710:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:2.0:1787828610.918712:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:2.0:1787828610.918714:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828610.918716:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:2.0:1787828610.918719:0:468:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787828610.918722:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828610.918724:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787828610.918727:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828610.918730:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828610.918732:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:2.0:1787828610.918734:0:468:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828610.918742:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787828610.918744:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828610.918746:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787828610.918748:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787828610.918752:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828610.918762:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787828610.918769:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2f0c0 x1874670646747008/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787828610.918777:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787828610.918781:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828610.918811:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787828610.918814:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828610.918818:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1787828610.918819:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787828610.918820:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787828610.918822:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787828610.918823:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:1.0:1787828610.918828:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00080000:2.0:1787828610.918828:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3411 00000100:00080000:2.0:1787828610.918831:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1787828610.918832:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1787828610.918836:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787828610.918837:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787828610.918838:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:1.0:1787828610.918840:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1787828610.918841:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787828610.918844:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 02000000:00000001:1.0:1787828610.918845:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000020:00000001:2.0:1787828610.918846:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 02000000:00000001:1.0:1787828610.918850:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000020:00000001:2.0:1787828610.918850:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:2.0:1787828610.918855:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 02000000:00000001:1.0:1787828610.918856:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828610.918857:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:2.0:1787828610.918863:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00100000:1.0:1787828610.918864:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646747008:192.168.203.143@tcp:250:kworker.0 00000100:00000001:2.0:1787828610.918865:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828610.918870:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1787828610.918872:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787828610.918876:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787828610.918877:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1787828610.918879:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828610.918881:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828610.918882:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 02000000:00000001:1.0:1787828610.918886:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000008:00100000:2.0:1787828610.918888:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1787828610.918890:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828610.918892:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828610.918892:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828610.918900:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787828610.918902:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828610.918903:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787828610.918904:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787828610.918906:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828610.918911:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787828610.918916:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2c3c0 x1874670646747136/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787828610.918922:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787828610.918923:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787828610.918927:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417880768 : -126234291670848 : ffff8d30c9e2f0c0) 00000100:00000001:2.0:1787828610.918928:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828610.918935:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:2.0:1787828610.918935:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787828610.918940:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828610.918942:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1787828610.918943:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787828610.918945:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787828610.918947:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787828610.918949:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3411 00000100:00080000:2.0:1787828610.918953:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787828610.918954:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828610.918955:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787828610.918957:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787828610.918959:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787828610.918960:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000020:00000001:2.0:1787828610.918962:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:2.0:1787828610.918964:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828610.918965:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:2.0:1787828610.918967:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000800:00000100:0.0:1787828610.918972:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:2.0:1787828610.918973:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828610.918977:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787828610.918979:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828610.918981:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828610.918982:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1787828610.918983:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000008:00100000:2.0:1787828610.918984:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1787828610.918987:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828610.918989:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828610.918989:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828610.918993:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828610.918995:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:0.0:1787828610.918996:0:2390:0:(events.c:68:request_out_callback()) Process leaving 02000000:00000001:2.0:1787828610.918997:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828610.919000:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787828610.919001:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:0.0:1787828610.919002:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828610.919003:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828610.919010:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4298085120 00000100:00000001:2.0:1787828610.919011:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787828610.919018:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2f480 x1874670646747264/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787828610.919020:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1787828610.919026:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4298085120 00000100:00000001:0.0:1787828610.919032:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 00000100:00000001:2.0:1787828610.919034:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787828610.919037:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787828610.919037:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828610.919040:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828610.919041:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787828610.919043:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1787828610.919045:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00080000:2.0:1787828610.919047:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1787828610.919049:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787828610.919051:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1787828610.919054:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828610/real 1787828610] req@ffff8d30c9e2f0c0 x1874670646747008/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828635 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787828610.919054:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787828610.919059:0:468:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3411 00000100:00080000:2.0:1787828610.919064:0:468:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787828610.919069:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000001:0.0:1787828610.919071:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000001:2.0:1787828610.919072:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000400:00000001:0.0:1787828610.919075:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000020:00000001:2.0:1787828610.919076:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787828610.919078:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:0.0:1787828610.919079:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000020:00000001:2.0:1787828610.919080:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:0.0:1787828610.919083:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000020:00000001:2.0:1787828610.919084:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:0.0:1787828610.919086:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828610.919088:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787828610.919089:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828610.919090:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:0.0:1787828610.919092:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:2.0:1787828610.919096:0:468:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00100000:0.0:1787828610.919099:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2f0c0 x1874670646747008/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828635 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787828610.919100:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828610.919104:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787828610.919107:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828610.919112:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828610.919113:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828610.919115:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1787828610.919118:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000008:00100000:2.0:1787828610.919118:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00000001:0.0:1787828610.919124:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000008:00000001:2.0:1787828610.919125:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828610.919130:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828610.919131:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828610.919133:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828610.919136:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 02000000:00000001:2.0:1787828610.919137:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:0.0:1787828610.919139:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 02000000:00000001:2.0:1787828610.919139:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787828610.919143:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00000001:2.0:1787828610.919144:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787828610.919147:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787828610.919150:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787828610.919151:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787828610.919154:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000001:2.0:1787828610.919155:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828610.919159:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646747008:192.168.203.143@tcp:250:kworker.0 00000100:00100000:2.0:1787828610.919163:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2c000 x1874670646747392/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787828610.919165:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828610.919170:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828610.919173:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 00000100:00000001:2.0:1787828610.919174:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 02000000:00000001:0.0:1787828610.919176:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 00000100:00000001:2.0:1787828610.919177:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828610.919181:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00000001:2.0:1787828610.919182:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787828610.919185:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787828610.919187:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787828610.919192:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787828610.919193:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787828610.919204:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787828610.919206:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828610.919209:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828610.919217:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787828610.919219:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828610.919222:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787828610.919223:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787828610.919226:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1787828610.919228:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787828610.919230:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828610.919233:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787828610.919236:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787828610.919239:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828610.919244:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646747136:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787828610.919249:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787828610.919252:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787828610.919254:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828610.919256:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1787828610.919260:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828610.919267:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787828610.919269:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417869248 : -126234291682368 : ffff8d30c9e2c3c0) 00000100:00000001:0.0:1787828610.919273:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1787828610.919284:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:0.0:1787828610.919288:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1787828610.919290:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828610.919291:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828610.919293:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787828610.919294:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828610.919298:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298083862 00000100:00000001:0.0:1787828610.919300:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298085120 00000100:00000001:0.0:1787828610.919302:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298083862 00000100:00000001:0.0:1787828610.919303:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298085120 00000100:00000001:0.0:1787828610.919305:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787828610.919306:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828610.919308:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787828610.919311:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828610/real 1787828610] req@ffff8d30c9e2c3c0 x1874670646747136/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828635 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787828610.919317:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787828610.919318:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787828610.919320:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787828610.919321:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787828610.919321:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828610.919322:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828610.919323:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787828610.919326:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2c3c0 x1874670646747136/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828635 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787828610.919330:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828610.919332:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787828610.919334:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828610.919335:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828610.919335:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828610.919336:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787828610.919338:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787828610.919339:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828610.919341:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787828610.919342:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787828610.919344:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646747136:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787828610.919347:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1787828610.919348:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787828610.919349:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828610.919351:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787828610.919352:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787828610.919353:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828610.919355:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646747264:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787828610.919359:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787828610.919360:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787828610.919361:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828610.919362:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1787828610.919364:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828610.919367:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787828610.919368:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417881728 : -126234291669888 : ffff8d30c9e2f480) 00000100:00000001:0.0:1787828610.919371:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1787828610.919377:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:0.0:1787828610.919379:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1787828610.919381:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828610.919382:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828610.919383:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787828610.919384:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828610.919386:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298083862 00000100:00000001:0.0:1787828610.919388:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298085120 00000100:00000001:0.0:1787828610.919390:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298083862 00000100:00000001:0.0:1787828610.919391:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298085120 00000100:00000001:0.0:1787828610.919392:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787828610.919394:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828610.919395:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787828610.919397:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828610/real 1787828610] req@ffff8d30c9e2f480 x1874670646747264/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828635 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787828610.919401:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787828610.919402:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787828610.919403:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787828610.919404:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787828610.919405:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828610.919407:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828610.919408:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787828610.919412:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2f480 x1874670646747264/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828635 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787828610.919424:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828610.919426:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787828610.919429:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828610.919430:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828610.919431:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828610.919433:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787828610.919435:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787828610.919438:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828610.919441:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787828610.919443:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787828610.919447:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646747264:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787828610.919451:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1787828610.919452:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787828610.919454:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828610.919456:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787828610.919458:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787828610.919460:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828610.919464:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646747392:192.168.203.143@tcp:8:kworker.0 00000100:00000001:0.0:1787828610.919467:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787828610.919469:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787828610.919470:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828610.919471:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1787828610.919473:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828610.919477:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787828610.919478:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417868288 : -126234291683328 : ffff8d30c9e2c000) 00000100:00000001:0.0:1787828610.919481:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1787828610.919489:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:0.0:1787828610.919493:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1787828610.919494:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828610.919495:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828610.919496:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787828610.919498:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828610.919500:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298083875 00000100:00000001:0.0:1787828610.919502:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298085120 00000100:00000001:0.0:1787828610.919503:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298083875 00000100:00000001:0.0:1787828610.919505:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298085120 00000100:00000001:0.0:1787828610.919506:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787828610.919507:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828610.919509:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787828610.919511:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828610/real 1787828610] req@ffff8d30c9e2c000 x1874670646747392/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828635 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787828610.919516:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787828610.919516:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787828610.919518:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787828610.919519:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787828610.919520:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828610.919521:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828610.919522:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787828610.919524:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2c000 x1874670646747392/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828635 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787828610.919528:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828610.919529:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787828610.919531:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828610.919532:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828610.919532:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828610.919534:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787828610.919535:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787828610.919536:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828610.919538:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787828610.919540:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787828610.919542:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646747392:192.168.203.143@tcp:8:kworker.0 00000100:00000001:0.0:1787828610.919544:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828610.919546:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828610.919547:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787828610.919548:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787828610.919549:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787828610.919550:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787828610.919551:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787828610.919552:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787828610.919553:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787828610.919555:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787828610.919555:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828610.919557:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828610.919557:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787828610.919558:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787828610.919559:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787828610.919560:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787828610.919561:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787828610.919562:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787828610.919563:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787828610.919564:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787828610.919564:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828610.919565:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828610.919566:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787828610.919567:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787828610.919568:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787828610.919568:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787828610.919569:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787828610.919570:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787828610.919571:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787828610.919572:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787828610.919573:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828610.919574:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828610.919577:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787828610.919577:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828610.919579:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787828610.919580:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828611.814735:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828611.814745:0:815:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1787828611.814747:0:815:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1787828611.814751:0:815:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828611.814756:0:815:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1787828611.814758:0:815:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828611.814760:0:815:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828611.814762:0:815:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828611.814840:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828611.814846:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828611.814850:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828612.838708:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828612.838715:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828612.838783:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828612.838787:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828612.838802:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828613.862778:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828613.862798:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828613.862860:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828613.862864:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828613.862867:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828614.886690:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828614.886698:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828614.886750:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828614.886754:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828614.886757:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828615.910743:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828615.910750:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828615.910832:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828615.910838:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828615.910842:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787828616.038726:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828616.038738:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1787828616.038741:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787828616.038745:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787828616.038750:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787828616.038757:0:468:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 3416 00000100:00080000:0.0:1787828616.038778:0:468:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787828616.038782:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787828616.038783:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787828616.038787:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787828616.038790:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787828616.038793:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:0.0:1787828616.038795:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:0.0:1787828616.038797:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787828616.038798:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:0.0:1787828616.038803:0:468:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787828616.038804:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828616.038807:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787828616.038810:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787828616.038813:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828616.038815:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1787828616.038818:0:468:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828616.038830:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787828616.038832:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828616.038835:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787828616.038838:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787828616.038841:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828616.038858:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828616.038868:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d27c00 x1874670646747520/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787828616.038877:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787828616.038881:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828616.038897:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787828616.038899:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828616.038904:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1787828616.038905:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787828616.038906:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787828616.038908:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787828616.038910:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3416 00000100:00080000:0.0:1787828616.038914:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787828616.038916:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787828616.038917:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787828616.038918:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787828616.038920:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787828616.038921:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000020:00000001:0.0:1787828616.038923:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:0.0:1787828616.038925:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828616.038926:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787828616.038927:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:0.0:1787828616.038932:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787828616.038933:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787828616.038934:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828616.038937:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:2.0:1787828616.038937:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1787828616.038941:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787828616.038943:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828616.038944:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787828616.038947:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828616.038949:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 02000000:00000001:2.0:1787828616.038950:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000008:00100000:0.0:1787828616.038953:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 02000000:00000001:2.0:1787828616.038954:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000008:00000001:0.0:1787828616.038960:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828616.038960:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828616.038964:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787828616.038967:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d27c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646747520:192.168.203.143@tcp:250:kworker.0 02000000:00000001:0.0:1787828616.038968:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787828616.038971:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828616.038974:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:0.0:1787828616.038975:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787828616.038978:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787828616.038980:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828616.038982:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:0.0:1787828616.038984:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828616.038986:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828616.038989:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00100000:0.0:1787828616.038992:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d252c0 x1874670646747648/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:2.0:1787828616.038996:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828616.039007:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787828616.039009:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828616.039012:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787828616.039014:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:2.0:1787828616.039015:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00080000:0.0:1787828616.039017:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1787828616.039018:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601351680 : -126234108199936 : ffff8d30d4d27c00) 00000100:00000001:0.0:1787828616.039021:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787828616.039024:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787828616.039024:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:0.0:1787828616.039029:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787828616.039031:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3416 00000100:00080000:0.0:1787828616.039034:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787828616.039035:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787828616.039036:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787828616.039037:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787828616.039038:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787828616.039040:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000020:00000001:0.0:1787828616.039041:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:0.0:1787828616.039043:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787828616.039044:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:0.0:1787828616.039046:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787828616.039047:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828616.039048:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787828616.039050:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787828616.039051:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828616.039053:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1787828616.039055:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1787828616.039057:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828616.039058:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828616.039061:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787828616.039062:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828616.039064:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787828616.039065:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787828616.039066:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828616.039070:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:2.0:1787828616.039070:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00100000:0.0:1787828616.039083:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d25680 x1874670646747776/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787828616.039087:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1787828616.039090:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787828616.039093:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828616.039093:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828616.039096:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828616.039097:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787828616.039099:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:2.0:1787828616.039101:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00080000:0.0:1787828616.039104:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1787828616.039106:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787828616.039107:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787828616.039109:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828616.039110:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787828616.039114:0:468:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3416 00000100:00000001:2.0:1787828616.039114:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4298090240 00000100:00080000:0.0:1787828616.039117:0:468:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787828616.039119:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787828616.039120:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787828616.039121:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787828616.039122:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787828616.039124:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000020:00000001:0.0:1787828616.039126:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:2.0:1787828616.039126:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1787828616.039131:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787828616.039132:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:2.0:1787828616.039135:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4298090240 00000100:00080000:0.0:1787828616.039136:0:468:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787828616.039141:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828616.039143:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:2.0:1787828616.039143:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 02000000:00000001:0.0:1787828616.039148:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787828616.039148:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787828616.039152:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828616.039153:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828616.039156:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:2.0:1787828616.039157:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000008:00100000:0.0:1787828616.039163:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787828616.039164:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828616/real 1787828616] req@ffff8d30d4d27c00 x1874670646747520/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828641 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00000001:0.0:1787828616.039170:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828616.039175:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828616.039181:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000400:00000001:2.0:1787828616.039183:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 02000000:00000001:0.0:1787828616.039184:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:2.0:1787828616.039186:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787828616.039187:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787828616.039190:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:2.0:1787828616.039190:0:2390:0:(events.c:78:reply_in_callback()) Process entered 02000000:00000001:0.0:1787828616.039194:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828616.039194:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787828616.039198:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828616.039199:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828616.039200:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828616.039204:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787828616.039207:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d26940 x1874670646747904/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787828616.039211:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d27c00 x1874670646747520/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828641 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787828616.039228:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787828616.039233:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828616.039237:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828616.039242:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828616.039247:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787828616.039249:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828616.039250:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828616.039251:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828616.039253:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787828616.039255:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787828616.039257:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828616.039260:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:2.0:1787828616.039262:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1787828616.039265:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d27c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646747520:192.168.203.143@tcp:250:kworker.0 00000100:00000001:2.0:1787828616.039269:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828616.039271:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828616.039272:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787828616.039275:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1787828616.039277:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787828616.039279:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:2.0:1787828616.039280:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:2.0:1787828616.039284:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787828616.039286:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1787828616.039289:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787828616.039290:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828616.039292:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828616.039300:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1787828616.039302:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828616.039303:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787828616.039304:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1787828616.039306:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1787828616.039307:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787828616.039308:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828616.039309:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787828616.039311:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828616.039312:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787828616.039314:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d252c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646747648:192.168.203.143@tcp:38:kworker.0 00000100:00000001:2.0:1787828616.039317:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787828616.039318:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1787828616.039319:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828616.039321:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1787828616.039323:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828616.039327:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787828616.039328:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601341120 : -126234108210496 : ffff8d30d4d252c0) 00000100:00000001:2.0:1787828616.039332:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1787828616.039340:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:2.0:1787828616.039344:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1787828616.039345:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828616.039346:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828616.039347:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1787828616.039349:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828616.039352:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298088982 00000100:00000001:2.0:1787828616.039354:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298090240 00000100:00000001:2.0:1787828616.039356:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298088982 00000100:00000001:2.0:1787828616.039358:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298090240 00000100:00000001:2.0:1787828616.039359:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787828616.039361:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828616.039362:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1787828616.039364:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828616/real 1787828616] req@ffff8d30d4d252c0 x1874670646747648/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828641 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1787828616.039369:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787828616.039370:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1787828616.039372:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1787828616.039373:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787828616.039374:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828616.039375:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828616.039376:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787828616.039379:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d252c0 x1874670646747648/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828641 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787828616.039388:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828616.039390:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787828616.039391:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828616.039392:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828616.039393:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828616.039394:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787828616.039395:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787828616.039397:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828616.039399:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:2.0:1787828616.039400:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1787828616.039402:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d252c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646747648:192.168.203.143@tcp:38:kworker.0 00000100:00000001:2.0:1787828616.039404:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1787828616.039405:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787828616.039406:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828616.039407:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787828616.039408:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828616.039409:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787828616.039411:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d25680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646747776:192.168.203.143@tcp:38:kworker.0 00000100:00000001:2.0:1787828616.039413:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787828616.039414:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1787828616.039415:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828616.039416:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1787828616.039418:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828616.039421:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787828616.039422:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601342080 : -126234108209536 : ffff8d30d4d25680) 00000100:00000001:2.0:1787828616.039425:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1787828616.039431:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:2.0:1787828616.039433:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1787828616.039435:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828616.039436:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828616.039437:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1787828616.039439:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828616.039441:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298088982 00000100:00000001:2.0:1787828616.039443:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298090240 00000100:00000001:2.0:1787828616.039445:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298088982 00000100:00000001:2.0:1787828616.039446:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298090240 00000100:00000001:2.0:1787828616.039447:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787828616.039449:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828616.039450:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1787828616.039452:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828616/real 1787828616] req@ffff8d30d4d25680 x1874670646747776/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828641 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1787828616.039457:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787828616.039457:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1787828616.039459:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1787828616.039459:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787828616.039460:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828616.039461:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828616.039462:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787828616.039464:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d25680 x1874670646747776/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828641 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787828616.039469:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828616.039470:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787828616.039472:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828616.039473:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828616.039474:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828616.039475:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787828616.039476:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787828616.039477:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828616.039479:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:2.0:1787828616.039480:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1787828616.039483:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d25680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646747776:192.168.203.143@tcp:38:kworker.0 00000100:00000001:2.0:1787828616.039486:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1787828616.039487:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787828616.039488:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828616.039490:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787828616.039492:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828616.039494:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787828616.039497:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d26940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646747904:192.168.203.143@tcp:8:kworker.0 00000100:00000001:2.0:1787828616.039501:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787828616.039503:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1787828616.039505:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828616.039506:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1787828616.039510:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828616.039515:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787828616.039517:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601346880 : -126234108204736 : ffff8d30d4d26940) 00000100:00000001:2.0:1787828616.039520:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1787828616.039532:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:2.0:1787828616.039537:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1787828616.039539:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828616.039541:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828616.039542:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1787828616.039545:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828616.039549:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298088995 00000100:00000001:2.0:1787828616.039552:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298090240 00000100:00000001:2.0:1787828616.039554:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298088995 00000100:00000001:2.0:1787828616.039556:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298090240 00000100:00000001:2.0:1787828616.039557:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787828616.039559:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828616.039560:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1787828616.039563:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828616/real 1787828616] req@ffff8d30d4d26940 x1874670646747904/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828641 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1787828616.039573:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787828616.039574:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1787828616.039576:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1787828616.039577:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787828616.039578:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828616.039579:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828616.039581:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787828616.039583:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d26940 x1874670646747904/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828641 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787828616.039587:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828616.039589:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787828616.039590:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828616.039591:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828616.039592:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828616.039593:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787828616.039594:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787828616.039596:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828616.039618:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:2.0:1787828616.039620:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1787828616.039622:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d26940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646747904:192.168.203.143@tcp:8:kworker.0 00000100:00000001:2.0:1787828616.039625:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828616.039626:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828616.039628:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787828616.039629:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1787828616.039631:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787828616.039632:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:2.0:1787828616.039633:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:2.0:1787828616.039634:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787828616.039635:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1787828616.039638:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787828616.039639:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828616.039640:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828616.039641:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787828616.039642:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1787828616.039643:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787828616.039644:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:2.0:1787828616.039645:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:2.0:1787828616.039646:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787828616.039646:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1787828616.039647:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787828616.039648:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828616.039649:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828616.039650:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787828616.039650:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1787828616.039651:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787828616.039652:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:2.0:1787828616.039652:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:2.0:1787828616.039653:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787828616.039654:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1787828616.039656:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787828616.039657:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828616.039658:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828616.039660:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1787828616.039661:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828616.039662:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787828616.039663:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828616.934706:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828616.934713:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828616.934789:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828616.934795:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828616.934798:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828617.958720:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828617.958729:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828617.958799:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828617.958802:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828617.958805:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828618.982679:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828618.982684:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828618.982736:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828618.982740:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828618.982744:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828620.006724:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828620.006733:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828620.006811:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828620.006815:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828620.006820:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828621.030679:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828621.030686:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828621.030766:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828621.030771:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828621.030775:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787828621.158780:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828621.158790:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1787828621.158794:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787828621.158797:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787828621.158802:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787828621.158809:0:468:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 3421 00000100:00080000:2.0:1787828621.158821:0:468:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787828621.158824:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828621.158826:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787828621.158830:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787828621.158833:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787828621.158836:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:2.0:1787828621.158838:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:2.0:1787828621.158839:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828621.158840:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:2.0:1787828621.158844:0:468:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787828621.158846:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828621.158849:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787828621.158852:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828621.158855:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828621.158857:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:2.0:1787828621.158859:0:468:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828621.158873:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787828621.158874:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828621.158878:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787828621.158880:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787828621.158885:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828621.158897:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787828621.158906:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2ed00 x1874670646748032/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787828621.158915:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787828621.158919:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828621.158940:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787828621.158943:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828621.158945:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1787828621.158946:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787828621.158947:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787828621.158950:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787828621.158952:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3421 00000100:00080000:2.0:1787828621.158954:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787828621.158957:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828621.158959:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787828621.158961:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787828621.158962:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787828621.158964:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000020:00000001:2.0:1787828621.158965:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:2.0:1787828621.158968:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828621.158970:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:2.0:1787828621.158972:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787828621.158974:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828621.158975:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787828621.158981:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828621.158984:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828621.158985:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787828621.158987:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1787828621.158993:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00000001:0.0:1787828621.158996:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000008:00000001:2.0:1787828621.158998:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828621.159002:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828621.159004:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 02000000:00000001:2.0:1787828621.159006:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:0.0:1787828621.159009:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 02000000:00000001:2.0:1787828621.159009:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828621.159011:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787828621.159015:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:0.0:1787828621.159016:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828621.159018:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828621.159026:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828621.159029:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000100:00100000:2.0:1787828621.159033:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2da40 x1874670646748160/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:0.0:1787828621.159036:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787828621.159047:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 02000000:00000001:0.0:1787828621.159049:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828621.159050:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828621.159064:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787828621.159070:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1787828621.159072:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646748032:192.168.203.143@tcp:250:kworker.0 00000100:00080000:2.0:1787828621.159077:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1787828621.159082:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787828621.159084:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787828621.159086:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787828621.159089:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3421 00000100:00000001:0.0:1787828621.159091:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00080000:2.0:1787828621.159093:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787828621.159096:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 02000000:00000001:0.0:1787828621.159097:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:2.0:1787828621.159097:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787828621.159101:0:468:0:(genops.c:792:class_conn2export()) Process entered 02000000:00000001:0.0:1787828621.159102:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1787828621.159104:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 02000000:00000001:0.0:1787828621.159106:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000020:00000001:2.0:1787828621.159108:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000020:00000001:2.0:1787828621.159111:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:2.0:1787828621.159113:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828621.159114:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 02000000:00000001:0.0:1787828621.159115:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787828621.159122:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787828621.159126:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828621.159128:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787828621.159130:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787828621.159132:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 02000000:00000001:2.0:1787828621.159132:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828621.159136:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417879808 : -126234291671808 : ffff8d30c9e2ed00) 00000100:00000001:2.0:1787828621.159137:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1787828621.159141:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00000001:0.0:1787828621.159143:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000008:00000001:2.0:1787828621.159149:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828621.159152:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828621.159157:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787828621.159157:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828621.159159:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787828621.159161:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787828621.159163:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828621.159167:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787828621.159178:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2c000 x1874670646748288/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787828621.159188:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:0.0:1787828621.159202:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1787828621.159202:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787828621.159207:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828621.159209:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828621.159211:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828621.159213:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787828621.159214:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1787828621.159222:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00080000:2.0:1787828621.159223:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1787828621.159227:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787828621.159230:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787828621.159232:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828621.159234:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:0.0:1787828621.159237:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4298095360 00000100:00080000:2.0:1787828621.159238:0:468:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3421 00000100:00080000:2.0:1787828621.159243:0:468:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787828621.159245:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828621.159246:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787828621.159247:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787828621.159248:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787828621.159254:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000020:00000001:2.0:1787828621.159257:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:2.0:1787828621.159259:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787828621.159260:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 00000100:00000001:2.0:1787828621.159261:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:2.0:1787828621.159268:0:468:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787828621.159272:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828621.159273:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4298095360 00000100:00000001:0.0:1787828621.159279:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1787828621.159281:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828621.159283:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:0.0:1787828621.159285:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828621.159290:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 02000000:00000001:2.0:1787828621.159290:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828621.159298:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828621.159300:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828621/real 1787828621] req@ffff8d30c9e2ed00 x1874670646748032/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828646 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787828621.159302:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1787828621.159307:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000008:00000001:2.0:1787828621.159319:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:0.0:1787828621.159320:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787828621.159327:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1787828621.159327:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828621.159330:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787828621.159334:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787828621.159336:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828621.159338:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 02000000:00000001:2.0:1787828621.159338:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:0.0:1787828621.159343:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:2.0:1787828621.159343:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828621.159354:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787828621.159358:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00100000:0.0:1787828621.159359:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2ed00 x1874670646748032/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828646 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1787828621.159362:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828621.159373:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828621.159375:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787828621.159381:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2f480 x1874670646748416/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787828621.159382:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787828621.159394:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828621.159402:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828621.159407:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787828621.159408:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828621.159413:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787828621.159413:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828621.159423:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787828621.159424:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787828621.159439:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828621.159447:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787828621.159451:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787828621.159456:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646748032:192.168.203.143@tcp:250:kworker.0 00000100:00000001:0.0:1787828621.159465:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828621.159468:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828621.159471:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787828621.159473:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787828621.159477:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787828621.159480:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787828621.159482:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787828621.159486:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787828621.159488:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787828621.159494:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787828621.159495:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828621.159498:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828621.159505:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787828621.159507:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828621.159510:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787828621.159512:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787828621.159513:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1787828621.159515:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787828621.159516:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828621.159519:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787828621.159522:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787828621.159524:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828621.159528:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2da40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646748160:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787828621.159532:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787828621.159534:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787828621.159536:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828621.159538:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1787828621.159541:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828621.159547:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787828621.159549:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417875008 : -126234291676608 : ffff8d30c9e2da40) 00000100:00000001:0.0:1787828621.159553:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1787828621.159568:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:0.0:1787828621.159573:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1787828621.159576:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828621.159577:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828621.159579:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787828621.159582:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828621.159586:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298094102 00000100:00000001:0.0:1787828621.159590:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298095360 00000100:00000001:0.0:1787828621.159593:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298094102 00000100:00000001:0.0:1787828621.159596:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298095360 00000100:00000001:0.0:1787828621.159623:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787828621.159626:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828621.159628:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787828621.159633:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828621/real 1787828621] req@ffff8d30c9e2da40 x1874670646748160/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828646 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787828621.159643:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787828621.159644:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787828621.159648:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787828621.159649:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787828621.159651:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828621.159653:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828621.159654:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787828621.159659:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2da40 x1874670646748160/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828646 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787828621.159666:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828621.159669:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787828621.159672:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828621.159673:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828621.159675:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828621.159677:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787828621.159678:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787828621.159681:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828621.159685:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787828621.159687:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787828621.159691:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2da40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646748160:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787828621.159695:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1787828621.159697:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787828621.159699:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828621.159702:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787828621.159704:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787828621.159706:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828621.159709:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646748288:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787828621.159713:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787828621.159716:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787828621.159717:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828621.159719:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1787828621.159723:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828621.159728:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787828621.159729:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417868288 : -126234291683328 : ffff8d30c9e2c000) 00000100:00000001:0.0:1787828621.159748:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1787828621.159763:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:0.0:1787828621.159769:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1787828621.159772:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828621.159774:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828621.159775:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787828621.159779:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828621.159783:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298094103 00000100:00000001:0.0:1787828621.159786:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298095360 00000100:00000001:0.0:1787828621.159788:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298094103 00000100:00000001:0.0:1787828621.159791:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298095360 00000100:00000001:0.0:1787828621.159793:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787828621.159795:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828621.159797:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787828621.159802:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828621/real 1787828621] req@ffff8d30c9e2c000 x1874670646748288/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828646 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787828621.159811:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787828621.159813:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787828621.159815:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787828621.159817:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787828621.159818:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828621.159821:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828621.159822:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787828621.159826:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2c000 x1874670646748288/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828646 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787828621.159834:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828621.159836:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787828621.159839:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828621.159840:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828621.159842:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828621.159844:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787828621.159845:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787828621.159848:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828621.159852:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787828621.159854:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787828621.159858:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646748288:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787828621.159862:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1787828621.159864:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787828621.159865:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828621.159868:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787828621.159870:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787828621.159872:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828621.159875:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646748416:192.168.203.143@tcp:8:kworker.0 00000100:00000001:0.0:1787828621.159879:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787828621.159881:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787828621.159883:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828621.159885:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1787828621.159888:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828621.159894:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787828621.159895:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417881728 : -126234291669888 : ffff8d30c9e2f480) 00000100:00000001:0.0:1787828621.159899:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1787828621.159910:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:0.0:1787828621.159915:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1787828621.159917:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828621.159918:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828621.159920:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787828621.159922:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828621.159925:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298094116 00000100:00000001:0.0:1787828621.159928:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298095360 00000100:00000001:0.0:1787828621.159930:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298094116 00000100:00000001:0.0:1787828621.159932:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298095360 00000100:00000001:0.0:1787828621.159934:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787828621.159936:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828621.159938:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787828621.159942:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828621/real 1787828621] req@ffff8d30c9e2f480 x1874670646748416/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828646 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787828621.159951:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787828621.159952:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787828621.159955:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787828621.159957:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787828621.159958:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828621.159959:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828621.159961:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787828621.159965:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2f480 x1874670646748416/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828646 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787828621.159973:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828621.159975:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787828621.159977:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828621.159979:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828621.159980:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828621.159982:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787828621.159984:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787828621.159987:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828621.159990:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787828621.159993:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787828621.159996:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646748416:192.168.203.143@tcp:8:kworker.0 00000100:00000001:0.0:1787828621.160000:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828621.160002:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828621.160003:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787828621.160005:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787828621.160008:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787828621.160009:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787828621.160010:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787828621.160013:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787828621.160014:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787828621.160017:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787828621.160018:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828621.160019:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828621.160020:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787828621.160022:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787828621.160023:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787828621.160025:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787828621.160026:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787828621.160028:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787828621.160029:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787828621.160031:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787828621.160033:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828621.160035:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828621.160036:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787828621.160037:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787828621.160039:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787828621.160040:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787828621.160041:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787828621.160043:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787828621.160044:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787828621.160046:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787828621.160048:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828621.160050:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828621.160054:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787828621.160055:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828621.160058:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787828621.160059:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828622.054718:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828622.054739:0:815:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1787828622.054741:0:815:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1787828622.054744:0:815:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828622.054750:0:815:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1787828622.054752:0:815:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828622.054753:0:815:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828622.054756:0:815:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828622.054816:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828622.054822:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828622.054827:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828623.078736:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828623.078744:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828623.078812:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828623.078818:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828623.078822:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828624.102697:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828624.102708:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828624.102781:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828624.102785:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828624.102788:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828625.126740:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828625.126751:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828625.126823:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828625.126827:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828625.126830:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828626.150694:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828626.150712:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828626.150770:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828626.150774:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828626.150777:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787828626.278665:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828626.278674:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1787828626.278676:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787828626.278680:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787828626.278683:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787828626.278689:0:468:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 3426 00000100:00080000:2.0:1787828626.278709:0:468:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787828626.278711:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828626.278713:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787828626.278716:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787828626.278719:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787828626.278721:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:2.0:1787828626.278723:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:2.0:1787828626.278725:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828626.278726:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:2.0:1787828626.278729:0:468:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787828626.278731:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828626.278732:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787828626.278735:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828626.278737:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828626.278739:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:2.0:1787828626.278741:0:468:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828626.278750:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787828626.278752:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828626.278754:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787828626.278756:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787828626.278759:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828626.278771:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787828626.278778:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2c3c0 x1874670646748544/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787828626.278787:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787828626.278789:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828626.278801:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787828626.278803:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828626.278806:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1787828626.278807:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787828626.278808:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787828626.278810:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787828626.278811:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3426 00000100:00080000:2.0:1787828626.278814:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787828626.278815:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828626.278816:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787828626.278818:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787828626.278819:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787828626.278820:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000020:00000001:2.0:1787828626.278822:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:2.0:1787828626.278823:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828626.278825:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:2.0:1787828626.278827:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787828626.278828:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828626.278830:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787828626.278834:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828626.278836:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828626.278837:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1787828626.278841:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1787828626.278843:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828626.278845:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828626.278846:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:2.0:1787828626.278847:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787828626.278851:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828626.278853:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1787828626.278853:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787828626.278858:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:0.0:1787828626.278859:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 02000000:00000001:2.0:1787828626.278861:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828626.278863:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787828626.278867:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828626.278869:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828626.278873:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787828626.278879:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00100000:2.0:1787828626.278879:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2f0c0 x1874670646748672/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:0.0:1787828626.278885:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828626.278895:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646748544:192.168.203.143@tcp:250:kworker.0 00000100:00000001:2.0:1787828626.278896:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787828626.278902:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828626.278906:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:2.0:1787828626.278908:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828626.278912:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000100:00080000:2.0:1787828626.278912:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 02000000:00000001:0.0:1787828626.278918:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787828626.278918:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1787828626.278921:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787828626.278924:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 02000000:00000001:0.0:1787828626.278926:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:2.0:1787828626.278926:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787828626.278931:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3426 02000000:00000001:0.0:1787828626.278934:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787828626.278935:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787828626.278939:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828626.278940:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787828626.278942:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787828626.278943:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787828626.278944:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000020:00000001:2.0:1787828626.278946:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:2.0:1787828626.278947:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828626.278948:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:2.0:1787828626.278950:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787828626.278951:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828626.278953:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 02000000:00000001:2.0:1787828626.278953:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:0.0:1787828626.278958:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417869248 : -126234291682368 : ffff8d30c9e2c3c0) 02000000:00000001:2.0:1787828626.278959:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828626.278965:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828626.278966:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:2.0:1787828626.278967:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1787828626.278971:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1787828626.278973:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828626.278975:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828626.278979:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787828626.278979:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828626.278980:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787828626.278982:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787828626.278983:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828626.278987:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787828626.278990:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2e940 x1874670646748800/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787828626.278996:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787828626.278997:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828626.278999:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787828626.279001:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828626.279004:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1787828626.279005:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787828626.279006:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787828626.279008:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787828626.279010:0:468:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3426 00000100:00080000:2.0:1787828626.279012:0:468:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787828626.279014:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828626.279014:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787828626.279016:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787828626.279017:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787828626.279018:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:0.0:1787828626.279019:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1787828626.279020:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:2.0:1787828626.279026:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828626.279027:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:0.0:1787828626.279028:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4298100480 00000100:00080000:2.0:1787828626.279030:0:468:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787828626.279035:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828626.279036:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787828626.279038:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828626.279039:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828626.279041:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1787828626.279043:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00000001:0.0:1787828626.279044:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 00000008:00000001:2.0:1787828626.279045:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828626.279050:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828626.279054:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:0.0:1787828626.279056:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4298100480 02000000:00000001:2.0:1787828626.279056:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828626.279058:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787828626.279061:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787828626.279063:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828626.279066:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 00000100:00000001:2.0:1787828626.279067:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787828626.279075:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2c780 x1874670646748928/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787828626.279076:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787828626.279084:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828626.279085:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787828626.279089:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828626.279092:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828626.279092:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828626.279100:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828626.279110:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787828626.279112:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:0.0:1787828626.279116:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787828626.279118:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787828626.279120:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1787828626.279121:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787828626.279123:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828626.279125:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787828626.279128:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787828626.279131:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828626.279137:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646748672:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787828626.279141:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787828626.279143:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787828626.279145:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828626.279148:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1787828626.279152:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828626.279159:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787828626.279161:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417880768 : -126234291670848 : ffff8d30c9e2f0c0) 00000100:00000001:0.0:1787828626.279167:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:0.0:1787828626.279182:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828626.279187:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298099222 00000100:00000001:0.0:1787828626.279190:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298100480 00000100:00000001:0.0:1787828626.279192:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298099222 00000100:00000001:0.0:1787828626.279194:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298100480 00000100:00000001:0.0:1787828626.279197:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787828626.279199:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828626.279201:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1787828626.279203:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787828626.279204:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828626.279207:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787828626.279209:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787828626.279211:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828626.279215:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646748800:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787828626.279219:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787828626.279221:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787828626.279222:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828626.279224:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1787828626.279227:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828626.279233:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787828626.279235:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417878848 : -126234291672768 : ffff8d30c9e2e940) 00000100:00000001:0.0:1787828626.279239:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:0.0:1787828626.279250:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828626.279254:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298099222 00000100:00000001:0.0:1787828626.279257:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298100480 00000100:00000001:0.0:1787828626.279259:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298099222 00000100:00000001:0.0:1787828626.279261:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298100480 00000100:00000001:0.0:1787828626.279264:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787828626.279266:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828626.279278:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1787828626.279280:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787828626.279282:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828626.279284:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787828626.279286:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787828626.279289:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828626.279293:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646748928:192.168.203.143@tcp:8:kworker.0 00000100:00000001:0.0:1787828626.279296:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787828626.279304:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787828626.279305:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828626.279307:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1787828626.279310:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828626.279317:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787828626.279318:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417870208 : -126234291681408 : ffff8d30c9e2c780) 00000100:00000001:0.0:1787828626.279322:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:0.0:1787828626.279333:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828626.279337:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298099235 00000100:00000001:0.0:1787828626.279339:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298100480 00000100:00000001:0.0:1787828626.279342:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298099235 00000100:00000001:0.0:1787828626.279344:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298100480 00000100:00000001:0.0:1787828626.279346:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787828626.279348:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828626.279351:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828626.279353:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828626.279357:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787828626.279359:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:0.0:1787828626.279361:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787828626.279363:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787828626.279365:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828626.279367:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828627.174761:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828627.174768:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828627.174838:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828627.174841:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828627.174844:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828628.198744:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828628.198750:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828628.198810:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828628.198815:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828628.198820:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828629.222698:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828629.222704:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828629.222757:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828629.222760:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828629.222763:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:3.0:1787828629.350931:0:2380:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.143:988 00000400:00000100:3.0:1787828629.350971:0:2380:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.143@tcp at host 192.168.203.143 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:3.0:1787828629.350984:0:2380:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.143@tcp 00000100:00000001:3.0:1787828629.350995:0:2380:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787828629.351013:0:2380:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787828629.351016:0:2380:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828629.351018:0:2380:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:3.0:1787828629.351029:0:2380:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.143@tcp 00000100:00000001:3.0:1787828629.351033:0:2380:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787828629.351036:0:2380:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787828629.351037:0:2380:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828629.351038:0:2380:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:3.0:1787828629.351042:0:2380:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.143@tcp 00000100:00000001:3.0:1787828629.351046:0:2380:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787828629.351049:0:2380:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787828629.351051:0:2380:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828629.351052:0:2380:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:3.0:1787828629.351056:0:2380:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.143@tcp 00000100:00000001:3.0:1787828629.351061:0:2380:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787828629.351064:0:2380:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787828629.351065:0:2380:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828629.351067:0:2380:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787828629.351181:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787828629.351188:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787828629.351191:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787828629.351203:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828626/real 1787828629] req@ffff8d30c9e2f0c0 x1874670646748672/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828651 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787828629.351218:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787828629.351220:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787828629.351227:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787828629.351231:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787828629.351233:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828629.351236:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828629.351238:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787828629.351243:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2f0c0 x1874670646748672/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828651 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787828629.351251:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828629.351255:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787828629.351258:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828629.351259:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828629.351260:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828629.351263:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787828629.351266:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787828629.351269:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828629.351274:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787828629.351277:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787828629.351281:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646748672:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787828629.351285:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787828629.351289:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828626/real 1787828629] req@ffff8d30c9e2e940 x1874670646748800/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828651 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787828629.351303:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787828629.351304:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787828629.351307:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787828629.351309:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787828629.351310:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828629.351312:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828629.351313:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787828629.351317:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2e940 x1874670646748800/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828651 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787828629.351323:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828629.351325:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787828629.351327:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828629.351329:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828629.351330:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828629.351332:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787828629.351333:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787828629.351336:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828629.351339:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787828629.351342:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787828629.351345:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646748800:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787828629.351349:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787828629.351353:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828626/real 1787828629] req@ffff8d30c9e2c780 x1874670646748928/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828651 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787828629.351362:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787828629.351363:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787828629.351367:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787828629.351368:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787828629.351370:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828629.351372:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828629.351374:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787828629.351378:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2c780 x1874670646748928/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828651 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787828629.351385:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828629.351387:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787828629.351389:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828629.351391:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828629.351392:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828629.351394:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787828629.351396:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787828629.351398:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828629.351402:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787828629.351405:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787828629.351408:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646748928:192.168.203.143@tcp:8:kworker.0 00000100:00000001:0.0:1787828629.351413:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787828629.351417:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828626/real 1787828629] req@ffff8d30c9e2c3c0 x1874670646748544/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828651 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787828629.351495:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787828629.351496:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787828629.351500:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787828629.351502:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787828629.351504:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828629.351506:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828629.351508:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787828629.351513:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2c3c0 x1874670646748544/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828651 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787828629.351520:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828629.351523:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787828629.351525:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828629.351528:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828629.351530:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828629.351532:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787828629.351534:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787828629.351536:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828629.351540:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787828629.351542:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787828629.351546:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646748544:192.168.203.143@tcp:250:kworker.0 00000100:00000001:0.0:1787828629.351550:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828629.351553:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828629.351555:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787828629.351557:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787828629.351562:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787828629.351565:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787828629.351568:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787828629.351573:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787828629.351575:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787828629.351581:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787828629.351582:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828629.351584:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828629.351585:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787828629.351586:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787828629.351588:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787828629.351589:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787828629.351591:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787828629.351592:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787828629.351593:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787828629.351597:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787828629.351598:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828629.351660:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828629.351661:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787828629.351663:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787828629.351667:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787828629.351669:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787828629.351671:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787828629.351673:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787828629.351686:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787828629.351689:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787828629.351692:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828629.351694:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828629.351696:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787828629.351697:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787828629.351701:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787828629.351702:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787828629.351703:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787828629.351706:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787828629.351708:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787828629.351711:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787828629.351713:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828629.351716:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828629.351726:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787828629.351729:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828629.351732:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787828629.351734:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828630.246705:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828630.246710:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828630.246764:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828630.246767:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828630.246770:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828631.270673:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828631.270678:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828631.270734:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828631.270738:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828631.270740:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787828631.398724:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828631.398732:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1787828631.398735:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787828631.398738:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787828631.398742:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787828631.398747:0:468:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 3431 00000100:00080000:2.0:1787828631.398756:0:468:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787828631.398758:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828631.398760:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787828631.398764:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787828631.398766:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787828631.398768:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:2.0:1787828631.398770:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:2.0:1787828631.398772:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828631.398773:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:2.0:1787828631.398777:0:468:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787828631.398778:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828631.398780:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787828631.398783:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828631.398785:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828631.398787:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:2.0:1787828631.398788:0:468:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828631.398796:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787828631.398797:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828631.398799:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787828631.398801:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787828631.398804:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828631.398813:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787828631.398820:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2de00 x1874670646749056/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787828631.398828:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787828631.398831:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828631.398841:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787828631.398843:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828631.398846:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1787828631.398847:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787828631.398848:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787828631.398850:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787828631.398851:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3431 00000100:00080000:2.0:1787828631.398853:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787828631.398854:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828631.398863:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787828631.398864:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787828631.398865:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787828631.398867:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000020:00000001:2.0:1787828631.398869:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:2.0:1787828631.398871:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828631.398874:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:2.0:1787828631.398877:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787828631.398879:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828631.398881:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787828631.398885:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828631.398887:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828631.398889:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1787828631.398891:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000008:00100000:2.0:1787828631.398893:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1787828631.398897:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828631.398898:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1787828631.398898:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828631.398903:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 02000000:00000001:2.0:1787828631.398904:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787828631.398908:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828631.398909:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787828631.398910:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:0.0:1787828631.398914:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828631.398914:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787828631.398916:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828631.398920:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787828631.398925:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787828631.398929:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828631.398929:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828631.398940:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646749056:192.168.203.143@tcp:250:kworker.0 00000100:00100000:2.0:1787828631.398942:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2fc00 x1874670646749184/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787828631.398950:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787828631.398955:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:2.0:1787828631.398958:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 02000000:00000001:0.0:1787828631.398960:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828631.398963:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828631.398965:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:2.0:1787828631.398969:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828631.398972:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787828631.398973:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828631.398980:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1787828631.398982:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787828631.398984:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787828631.398987:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:0.0:1787828631.398990:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00080000:2.0:1787828631.398990:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3431 00000100:00000001:0.0:1787828631.398994:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417875968 : -126234291675648 : ffff8d30c9e2de00) 00000100:00080000:2.0:1787828631.398999:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787828631.399001:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:2.0:1787828631.399005:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828631.399008:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787828631.399012:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787828631.399013:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787828631.399016:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000020:00000001:2.0:1787828631.399018:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:2.0:1787828631.399021:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828631.399022:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:2.0:1787828631.399025:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787828631.399034:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828631.399036:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787828631.399040:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000800:00000100:0.0:1787828631.399044:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 02000000:00000001:2.0:1787828631.399046:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828631.399050:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1787828631.399052:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1787828631.399055:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828631.399058:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1787828631.399058:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828631.399066:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787828631.399067:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828631.399069:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787828631.399070:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787828631.399071:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828631.399076:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828631.399078:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828631.399083:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787828631.399084:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2d2c0 x1874670646749312/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787828631.399086:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787828631.399097:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828631.399097:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787828631.399102:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828631.399105:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4298105600 00000100:00000001:2.0:1787828631.399107:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787828631.399112:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828631.399116:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1787828631.399118:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:0.0:1787828631.399119:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 00000100:00080000:2.0:1787828631.399120:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787828631.399125:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:0.0:1787828631.399128:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4298105600 00000100:00080000:2.0:1787828631.399129:0:468:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3431 00000100:00000001:0.0:1787828631.399137:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 00000100:00080000:2.0:1787828631.399137:0:468:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787828631.399141:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787828631.399144:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787828631.399144:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787828631.399151:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:0.0:1787828631.399152:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1787828631.399153:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787828631.399158:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000020:00000001:2.0:1787828631.399160:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:2.0:1787828631.399162:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787828631.399164:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00000001:2.0:1787828631.399164:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:2.0:1787828631.399171:0:468:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00100000:0.0:1787828631.399175:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828631/real 1787828631] req@ffff8d30c9e2de00 x1874670646749056/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828656 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787828631.399176:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828631.399181:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787828631.399184:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828631.399186:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828631.399189:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1787828631.399191:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000008:00000001:2.0:1787828631.399193:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828631.399198:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828631.399205:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787828631.399207:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828631.399211:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000400:00000001:0.0:1787828631.399212:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 02000000:00000001:2.0:1787828631.399213:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000400:00000001:0.0:1787828631.399216:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 02000000:00000001:2.0:1787828631.399218:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828631.399222:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1787828631.399225:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828631.399227:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787828631.399231:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828631.399235:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00100000:2.0:1787828631.399235:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2cb40 x1874670646749440/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787828631.399239:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787828631.399250:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2de00 x1874670646749056/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828656 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787828631.399251:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787828631.399256:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828631.399263:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828631.399271:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828631.399275:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787828631.399278:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828631.399280:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828631.399281:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828631.399284:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787828631.399287:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787828631.399290:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828631.399295:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787828631.399298:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787828631.399303:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646749056:192.168.203.143@tcp:250:kworker.0 00000100:00000001:0.0:1787828631.399308:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828631.399311:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828631.399312:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787828631.399315:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787828631.399318:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787828631.399320:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787828631.399323:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787828631.399327:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787828631.399329:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787828631.399334:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787828631.399336:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828631.399339:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828631.399345:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787828631.399348:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828631.399350:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787828631.399352:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787828631.399353:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1787828631.399355:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787828631.399356:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828631.399359:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787828631.399361:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787828631.399363:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828631.399368:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646749184:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787828631.399371:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787828631.399373:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787828631.399376:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828631.399378:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1787828631.399381:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828631.399387:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787828631.399389:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417883648 : -126234291667968 : ffff8d30c9e2fc00) 00000100:00000001:0.0:1787828631.399394:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1787828631.399409:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:0.0:1787828631.399414:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1787828631.399417:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828631.399418:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828631.399420:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787828631.399423:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828631.399428:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298104342 00000100:00000001:0.0:1787828631.399431:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298105600 00000100:00000001:0.0:1787828631.399434:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298104342 00000100:00000001:0.0:1787828631.399436:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298105600 00000100:00000001:0.0:1787828631.399438:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787828631.399441:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828631.399443:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787828631.399448:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828631/real 1787828631] req@ffff8d30c9e2fc00 x1874670646749184/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828656 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787828631.399457:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787828631.399459:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787828631.399461:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787828631.399463:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787828631.399464:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828631.399466:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828631.399468:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787828631.399472:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2fc00 x1874670646749184/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828656 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787828631.399478:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828631.399481:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787828631.399484:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828631.399485:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828631.399487:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828631.399488:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787828631.399490:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787828631.399493:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828631.399497:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787828631.399499:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787828631.399503:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646749184:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787828631.399508:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1787828631.399509:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787828631.399511:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828631.399513:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787828631.399515:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787828631.399517:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828631.399520:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2d2c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646749312:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787828631.399524:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787828631.399526:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787828631.399527:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828631.399529:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1787828631.399532:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828631.399537:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787828631.399539:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417873088 : -126234291678528 : ffff8d30c9e2d2c0) 00000100:00000001:0.0:1787828631.399543:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1787828631.399555:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:0.0:1787828631.399560:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1787828631.399562:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828631.399564:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828631.399565:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787828631.399568:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828631.399572:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298104342 00000100:00000001:0.0:1787828631.399575:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298105600 00000100:00000001:0.0:1787828631.399577:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298104342 00000100:00000001:0.0:1787828631.399580:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298105600 00000100:00000001:0.0:1787828631.399582:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787828631.399584:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828631.399586:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787828631.399591:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828631/real 1787828631] req@ffff8d30c9e2d2c0 x1874670646749312/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828656 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787828631.399643:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787828631.399645:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787828631.399649:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787828631.399650:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787828631.399652:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828631.399654:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828631.399656:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787828631.399674:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2d2c0 x1874670646749312/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828656 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787828631.399682:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828631.399685:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787828631.399688:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828631.399689:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828631.399691:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828631.399693:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787828631.399695:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787828631.399697:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828631.399701:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787828631.399704:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787828631.399707:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2d2c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646749312:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787828631.399732:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1787828631.399734:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787828631.399736:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828631.399739:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787828631.399741:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787828631.399743:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828631.399747:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646749440:192.168.203.143@tcp:8:kworker.0 00000100:00000001:0.0:1787828631.399750:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787828631.399752:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787828631.399754:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828631.399756:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1787828631.399759:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828631.399765:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787828631.399767:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417871168 : -126234291680448 : ffff8d30c9e2cb40) 00000100:00000001:0.0:1787828631.399770:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1787828631.399784:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:0.0:1787828631.399790:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1787828631.399792:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828631.399794:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828631.399796:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787828631.399799:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828631.399811:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298104356 00000100:00000001:0.0:1787828631.399814:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298105600 00000100:00000001:0.0:1787828631.399816:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298104356 00000100:00000001:0.0:1787828631.399818:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298105600 00000100:00000001:0.0:1787828631.399821:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787828631.399823:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828631.399825:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787828631.399830:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828631/real 1787828631] req@ffff8d30c9e2cb40 x1874670646749440/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828656 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787828631.399839:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787828631.399840:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787828631.399843:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787828631.399845:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787828631.399846:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828631.399848:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828631.399850:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787828631.399854:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2cb40 x1874670646749440/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828656 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787828631.399861:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828631.399864:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787828631.399866:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828631.399868:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828631.399869:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828631.399871:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787828631.399873:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787828631.399875:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828631.399879:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787828631.399881:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787828631.399884:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646749440:192.168.203.143@tcp:8:kworker.0 00000100:00000001:0.0:1787828631.399888:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828631.399890:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828631.399892:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787828631.399894:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787828631.399896:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787828631.399898:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787828631.399900:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787828631.399902:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787828631.399904:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787828631.399907:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787828631.399908:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828631.399910:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828631.399911:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787828631.399913:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787828631.399914:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787828631.399916:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787828631.399917:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787828631.399919:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787828631.399920:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787828631.399922:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787828631.399923:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828631.399925:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828631.399926:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787828631.399928:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787828631.399930:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787828631.399931:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787828631.399932:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787828631.399934:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787828631.399934:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787828631.399936:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787828631.399938:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828631.399940:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828631.399943:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787828631.399946:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828631.399948:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787828631.399950:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828632.294690:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828632.294699:0:815:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1787828632.294701:0:815:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1787828632.294705:0:815:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828632.294709:0:815:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1787828632.294712:0:815:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828632.294714:0:815:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828632.294715:0:815:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828632.294767:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828632.294770:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828632.294773:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828633.318726:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828633.318731:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828633.318796:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828633.318800:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828633.318802:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828634.342708:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828634.342713:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828634.342764:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828634.342769:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828634.342780:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828635.366752:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828635.366762:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828635.366835:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828635.366840:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828635.366843:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828636.390796:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828636.390821:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828636.390898:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828636.390906:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828636.390912:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787828636.518804:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828636.518816:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1787828636.518819:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787828636.518832:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787828636.518838:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787828636.518859:0:468:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 3437 00000100:00080000:0.0:1787828636.518873:0:468:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787828636.518893:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787828636.518895:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787828636.518901:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787828636.518903:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787828636.518907:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:0.0:1787828636.518909:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:0.0:1787828636.518911:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787828636.518912:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:0.0:1787828636.518917:0:468:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787828636.518920:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828636.518922:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787828636.518926:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787828636.518929:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828636.518931:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1787828636.518934:0:468:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828636.518949:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787828636.518950:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828636.518954:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787828636.518957:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787828636.518962:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828636.518978:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828636.518989:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d26940 x1874670646749568/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787828636.519000:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787828636.519005:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828636.519027:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787828636.519030:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828636.519033:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1787828636.519034:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787828636.519036:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787828636.519038:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787828636.519040:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3437 00000100:00080000:0.0:1787828636.519043:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787828636.519045:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787828636.519046:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787828636.519048:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787828636.519049:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787828636.519061:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000020:00000001:0.0:1787828636.519063:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:0.0:1787828636.519065:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787828636.519067:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:0.0:1787828636.519069:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787828636.519071:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828636.519072:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787828636.519078:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787828636.519080:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828636.519082:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1787828636.519088:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1787828636.519091:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828636.519093:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828636.519096:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787828636.519098:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828636.519099:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787828636.519100:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787828636.519102:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828636.519106:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787828636.519107:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828636.519120:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00100000:0.0:1787828636.519121:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d25680 x1874670646749696/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787828636.519125:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1787828636.519131:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787828636.519137:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828636.519138:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787828636.519143:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828636.519148:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:0.0:1787828636.519150:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828636.519164:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00080000:0.0:1787828636.519180:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 02000000:00000001:2.0:1787828636.519181:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787828636.519194:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1787828636.519197:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787828636.519200:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1787828636.519201:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d26940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646749568:192.168.203.143@tcp:250:kworker.0 00000100:00000001:0.0:1787828636.519207:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:2.0:1787828636.519214:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00080000:0.0:1787828636.519221:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3437 02000000:00000001:2.0:1787828636.519222:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1787828636.519231:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828636.519236:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00080000:0.0:1787828636.519240:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 02000000:00000001:2.0:1787828636.519245:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828636.519246:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787828636.519255:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787828636.519261:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787828636.519265:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787828636.519284:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:2.0:1787828636.519288:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000020:00000001:0.0:1787828636.519289:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:2.0:1787828636.519291:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601346880 : -126234108204736 : ffff8d30d4d26940) 00000100:00000001:0.0:1787828636.519299:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828636.519300:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:0.0:1787828636.519308:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:0.0:1787828636.519317:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787828636.519319:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828636.519322:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787828636.519327:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787828636.519329:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828636.519331:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1787828636.519333:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1787828636.519337:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828636.519339:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828636.519344:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787828636.519345:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:2.0:1787828636.519352:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:0.0:1787828636.519353:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787828636.519356:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787828636.519359:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828636.519368:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1787828636.519373:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828636.519376:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828636.519379:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787828636.519382:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828636.519385:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828636.519394:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d252c0 x1874670646749824/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787828636.519396:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4298110720 00000100:00000001:0.0:1787828636.519416:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787828636.519417:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1787828636.519422:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828636.519427:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828636.519428:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4298110720 00000100:00080000:0.0:1787828636.519430:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:2.0:1787828636.519435:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 00000100:00080000:0.0:1787828636.519437:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1787828636.519444:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:2.0:1787828636.519446:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00080000:0.0:1787828636.519447:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787828636.519455:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:2.0:1787828636.519457:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828636.519464:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00080000:0.0:1787828636.519465:0:468:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3437 00000100:00080000:0.0:1787828636.519477:0:468:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787828636.519482:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00100000:2.0:1787828636.519482:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828636/real 1787828636] req@ffff8d30d4d26940 x1874670646749568/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828661 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787828636.519505:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787828636.519517:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787828636.519520:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787828636.519527:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000400:00000001:2.0:1787828636.519530:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000020:00000001:0.0:1787828636.519539:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000400:00000001:2.0:1787828636.519539:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787828636.519549:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787828636.519554:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:2.0:1787828636.519556:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1787828636.519568:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00080000:0.0:1787828636.519570:0:468:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787828636.519574:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828636.519575:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828636.519594:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:2.0:1787828636.519594:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 02000000:00000001:0.0:1787828636.519603:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787828636.519607:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828636.519609:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:2.0:1787828636.519653:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787828636.519665:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d26940 x1874670646749568/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828661 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787828636.519701:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828636.519707:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787828636.519711:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828636.519715:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828636.519718:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000008:00100000:0.0:1787828636.519720:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00000001:2.0:1787828636.519723:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787828636.519728:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000008:00000001:0.0:1787828636.519732:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787828636.519735:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00000001:0.0:1787828636.519749:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787828636.519750:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:2.0:1787828636.519757:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1787828636.519765:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d26940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646749568:192.168.203.143@tcp:250:kworker.0 02000000:00000001:0.0:1787828636.519768:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787828636.519774:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828636.519779:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828636.519781:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787828636.519785:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:2.0:1787828636.519790:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 02000000:00000001:0.0:1787828636.519792:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828636.519814:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828636.519815:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787828636.519822:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 00000100:00100000:0.0:1787828636.519824:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d27c00 x1874670646749952/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:2.0:1787828636.519832:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00000001:0.0:1787828636.519834:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787828636.519838:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1787828636.519839:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:2.0:1787828636.519848:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 00000100:00000001:0.0:1787828636.519849:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828636.519893:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787828636.519896:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1787828636.519902:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787828636.519905:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828636.519909:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828636.519919:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1787828636.519921:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828636.519924:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787828636.519927:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1787828636.519929:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1787828636.519931:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787828636.519933:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828636.519936:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787828636.519939:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828636.519942:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787828636.519948:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d25680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646749696:192.168.203.143@tcp:38:kworker.0 00000100:00000001:2.0:1787828636.519952:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787828636.519955:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1787828636.519957:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828636.519959:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1787828636.519963:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828636.519972:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787828636.519973:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601342080 : -126234108209536 : ffff8d30d4d25680) 00000100:00000001:2.0:1787828636.519979:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1787828636.519997:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:2.0:1787828636.520003:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1787828636.520005:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828636.520007:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828636.520009:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1787828636.520012:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828636.520018:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298109463 00000100:00000001:2.0:1787828636.520022:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298110720 00000100:00000001:2.0:1787828636.520026:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298109463 00000100:00000001:2.0:1787828636.520030:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298110720 00000100:00000001:2.0:1787828636.520034:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787828636.520037:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828636.520040:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1787828636.520063:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828636/real 1787828636] req@ffff8d30d4d25680 x1874670646749696/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828661 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1787828636.520083:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787828636.520085:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1787828636.520089:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1787828636.520092:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787828636.520093:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828636.520096:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828636.520098:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787828636.520103:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d25680 x1874670646749696/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828661 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787828636.520113:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828636.520115:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787828636.520117:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828636.520119:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828636.520120:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828636.520122:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787828636.520124:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787828636.520126:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828636.520129:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:2.0:1787828636.520131:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1787828636.520134:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d25680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646749696:192.168.203.143@tcp:38:kworker.0 00000100:00000001:2.0:1787828636.520138:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1787828636.520139:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787828636.520141:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828636.520143:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787828636.520144:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828636.520146:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787828636.520148:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d252c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646749824:192.168.203.143@tcp:38:kworker.0 00000100:00000001:2.0:1787828636.520152:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787828636.520154:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1787828636.520156:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828636.520157:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1787828636.520160:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828636.520165:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787828636.520166:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601341120 : -126234108210496 : ffff8d30d4d252c0) 00000100:00000001:2.0:1787828636.520170:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1787828636.520181:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:2.0:1787828636.520186:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1787828636.520187:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828636.520189:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828636.520190:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1787828636.520192:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828636.520195:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298109463 00000100:00000001:2.0:1787828636.520208:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298110720 00000100:00000001:2.0:1787828636.520210:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298109463 00000100:00000001:2.0:1787828636.520212:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298110720 00000100:00000001:2.0:1787828636.520214:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787828636.520216:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828636.520217:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1787828636.520220:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828636/real 1787828636] req@ffff8d30d4d252c0 x1874670646749824/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828661 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1787828636.520226:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787828636.520227:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1787828636.520229:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1787828636.520230:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787828636.520231:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828636.520233:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828636.520234:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787828636.520237:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d252c0 x1874670646749824/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828661 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787828636.520242:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828636.520244:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787828636.520247:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828636.520248:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828636.520249:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828636.520250:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787828636.520252:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787828636.520254:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828636.520256:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:2.0:1787828636.520258:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1787828636.520260:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d252c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646749824:192.168.203.143@tcp:38:kworker.0 00000100:00000001:2.0:1787828636.520263:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1787828636.520264:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787828636.520265:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828636.520267:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787828636.520268:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828636.520270:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787828636.520272:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d27c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646749952:192.168.203.143@tcp:8:kworker.0 00000100:00000001:2.0:1787828636.520275:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787828636.520276:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1787828636.520278:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828636.520279:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1787828636.520281:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828636.520285:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787828636.520286:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601351680 : -126234108199936 : ffff8d30d4d27c00) 00000100:00000001:2.0:1787828636.520288:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1787828636.520295:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:2.0:1787828636.520298:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1787828636.520300:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828636.520301:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828636.520302:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1787828636.520304:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828636.520307:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298109476 00000100:00000001:2.0:1787828636.520309:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298110720 00000100:00000001:2.0:1787828636.520311:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298109476 00000100:00000001:2.0:1787828636.520313:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298110720 00000100:00000001:2.0:1787828636.520315:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787828636.520316:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828636.520318:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1787828636.520321:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828636/real 1787828636] req@ffff8d30d4d27c00 x1874670646749952/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828661 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1787828636.520327:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787828636.520327:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1787828636.520329:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1787828636.520330:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787828636.520331:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828636.520332:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828636.520333:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787828636.520336:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d27c00 x1874670646749952/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828661 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787828636.520341:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828636.520343:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787828636.520344:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828636.520346:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828636.520347:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828636.520348:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787828636.520350:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787828636.520352:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828636.520354:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:2.0:1787828636.520356:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1787828636.520358:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d27c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646749952:192.168.203.143@tcp:8:kworker.0 00000100:00000001:2.0:1787828636.520361:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828636.520362:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828636.520364:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787828636.520365:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1787828636.520367:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787828636.520369:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:2.0:1787828636.520370:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:2.0:1787828636.520372:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787828636.520373:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1787828636.520376:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787828636.520378:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828636.520379:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828636.520380:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787828636.520381:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1787828636.520383:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787828636.520384:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:2.0:1787828636.520385:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:2.0:1787828636.520386:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787828636.520387:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1787828636.520388:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787828636.520389:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828636.520390:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828636.520391:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787828636.520392:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1787828636.520393:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787828636.520393:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:2.0:1787828636.520394:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:2.0:1787828636.520395:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787828636.520396:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1787828636.520399:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787828636.520400:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828636.520401:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828636.520404:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1787828636.520405:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828636.520406:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787828636.520408:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828637.414741:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828637.414748:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828637.414848:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828637.414856:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828637.414861:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828638.438757:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828638.438765:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828638.438838:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828638.438842:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828638.438845:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828639.462734:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828639.462739:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828639.462794:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828639.462797:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828639.462800:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828640.486671:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828640.486677:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828640.486724:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828640.486735:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828640.486737:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828641.510721:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828641.510730:0:815:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1787828641.510733:0:815:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1787828641.510736:0:815:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828641.510741:0:815:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1787828641.510744:0:815:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828641.510747:0:815:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828641.510749:0:815:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828641.510799:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828641.510803:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828641.510805:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787828641.638688:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828641.638695:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1787828641.638697:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787828641.638700:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787828641.638704:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787828641.638709:0:468:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 3442 00000100:00080000:2.0:1787828641.638717:0:468:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787828641.638719:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828641.638721:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787828641.638724:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787828641.638726:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787828641.638728:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:2.0:1787828641.638730:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:2.0:1787828641.638732:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828641.638733:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:2.0:1787828641.638736:0:468:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787828641.638738:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828641.638740:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787828641.638742:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828641.638744:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828641.638746:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:2.0:1787828641.638748:0:468:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828641.638757:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787828641.638758:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828641.638761:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787828641.638763:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787828641.638766:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828641.638775:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787828641.638783:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2d680 x1874670646750080/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787828641.638791:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787828641.638794:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828641.638808:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787828641.638811:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828641.638813:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1787828641.638814:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787828641.638815:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787828641.638817:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787828641.638819:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3442 00000100:00080000:2.0:1787828641.638821:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787828641.638823:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828641.638824:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787828641.638826:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787828641.638828:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787828641.638829:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000020:00000001:2.0:1787828641.638831:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:2.0:1787828641.638832:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828641.638833:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:2.0:1787828641.638835:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787828641.638836:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828641.638838:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:2.0:1787828641.638838:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787828641.638842:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787828641.638845:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 02000000:00000001:2.0:1787828641.638846:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828641.638849:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1787828641.638850:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000008:00100000:2.0:1787828641.638853:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00000001:0.0:1787828641.638855:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000008:00000001:2.0:1787828641.638858:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828641.638859:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828641.638862:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828641.638866:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787828641.638866:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787828641.638870:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828641.638872:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787828641.638872:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787828641.638875:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787828641.638879:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828641.638879:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828641.638886:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828641.638890:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2d680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646750080:192.168.203.143@tcp:250:kworker.0 00000100:00100000:2.0:1787828641.638892:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2e1c0 x1874670646750208/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787828641.638900:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:2.0:1787828641.638904:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787828641.638906:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828641.638908:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:2.0:1787828641.638909:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828641.638911:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787828641.638913:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 02000000:00000001:0.0:1787828641.638916:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00080000:2.0:1787828641.638917:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1787828641.638920:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787828641.638921:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 02000000:00000001:0.0:1787828641.638923:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828641.638923:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787828641.638928:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3442 00000100:00080000:2.0:1787828641.638931:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787828641.638932:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828641.638933:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787828641.638934:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787828641.638935:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787828641.638936:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000020:00000001:2.0:1787828641.638938:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:0.0:1787828641.638939:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787828641.638940:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828641.638943:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:0.0:1787828641.638945:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417874048 : -126234291677568 : ffff8d30c9e2d680) 00000100:00080000:2.0:1787828641.638947:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787828641.638951:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828641.638953:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 02000000:00000001:2.0:1787828641.638953:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787828641.638957:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828641.638960:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828641.638961:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1787828641.638962:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1787828641.638964:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828641.638965:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828641.638969:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787828641.638970:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828641.638972:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787828641.638973:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787828641.638974:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828641.638977:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787828641.638979:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2cb40 x1874670646750336/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787828641.638984:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787828641.638986:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828641.638987:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787828641.638989:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828641.638991:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1787828641.638994:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000800:00000100:0.0:1787828641.638997:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00080000:2.0:1787828641.638997:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787828641.639003:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787828641.639005:0:468:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3442 00000100:00080000:2.0:1787828641.639008:0:468:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787828641.639009:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828641.639010:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787828641.639011:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787828641.639012:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:0.0:1787828641.639014:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000020:00000001:2.0:1787828641.639014:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000020:00000001:2.0:1787828641.639018:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:2.0:1787828641.639020:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787828641.639021:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828641.639022:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:0.0:1787828641.639026:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787828641.639027:0:468:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787828641.639031:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828641.639032:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787828641.639033:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828641.639035:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828641.639036:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1787828641.639037:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00000001:0.0:1787828641.639040:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000008:00000001:2.0:1787828641.639040:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828641.639044:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828641.639047:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787828641.639048:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828641.639049:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787828641.639050:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:0.0:1787828641.639051:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828641.639052:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828641.639058:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828641.639060:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4298115840 00000100:00100000:2.0:1787828641.639061:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2d2c0 x1874670646750464/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787828641.639069:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787828641.639070:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828641.639071:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828641.639073:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1787828641.639082:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4298115840 00000100:00000001:0.0:1787828641.639087:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1787828641.639092:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787828641.639094:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828641.639096:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787828641.639102:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828641/real 1787828641] req@ffff8d30c9e2d680 x1874670646750080/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828666 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787828641.639110:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787828641.639118:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787828641.639121:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787828641.639123:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787828641.639124:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828641.639126:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828641.639127:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787828641.639131:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2d680 x1874670646750080/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828666 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787828641.639136:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828641.639139:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787828641.639141:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828641.639142:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828641.639142:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828641.639144:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787828641.639145:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787828641.639147:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828641.639150:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787828641.639151:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787828641.639154:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2d680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646750080:192.168.203.143@tcp:250:kworker.0 00000100:00000001:0.0:1787828641.639158:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828641.639160:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828641.639161:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787828641.639163:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787828641.639166:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787828641.639169:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787828641.639171:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787828641.639175:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787828641.639177:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787828641.639180:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787828641.639182:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828641.639185:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828641.639191:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787828641.639193:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828641.639195:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787828641.639197:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787828641.639199:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1787828641.639200:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787828641.639201:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828641.639204:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787828641.639206:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787828641.639208:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828641.639212:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646750208:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787828641.639214:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787828641.639216:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787828641.639217:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828641.639219:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1787828641.639221:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828641.639225:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787828641.639226:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417876928 : -126234291674688 : ffff8d30c9e2e1c0) 00000100:00000001:0.0:1787828641.639230:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1787828641.639241:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:0.0:1787828641.639244:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1787828641.639246:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828641.639247:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828641.639248:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787828641.639250:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828641.639253:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298114582 00000100:00000001:0.0:1787828641.639255:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298115840 00000100:00000001:0.0:1787828641.639257:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298114582 00000100:00000001:0.0:1787828641.639259:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298115840 00000100:00000001:0.0:1787828641.639260:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787828641.639262:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828641.639263:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787828641.639266:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828641/real 1787828641] req@ffff8d30c9e2e1c0 x1874670646750208/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828666 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787828641.639271:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787828641.639272:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787828641.639273:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787828641.639275:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787828641.639276:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828641.639277:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828641.639278:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787828641.639280:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2e1c0 x1874670646750208/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828666 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787828641.639285:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828641.639287:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787828641.639288:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828641.639289:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828641.639290:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828641.639292:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787828641.639293:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787828641.639294:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828641.639296:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787828641.639298:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787828641.639299:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646750208:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787828641.639302:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1787828641.639303:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787828641.639304:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828641.639305:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787828641.639306:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787828641.639308:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828641.639309:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646750336:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787828641.639311:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787828641.639312:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787828641.639313:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828641.639314:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1787828641.639316:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828641.639319:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787828641.639320:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417871168 : -126234291680448 : ffff8d30c9e2cb40) 00000100:00000001:0.0:1787828641.639323:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1787828641.639329:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:0.0:1787828641.639331:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1787828641.639333:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828641.639334:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828641.639335:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787828641.639337:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828641.639339:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298114582 00000100:00000001:0.0:1787828641.639341:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298115840 00000100:00000001:0.0:1787828641.639342:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298114582 00000100:00000001:0.0:1787828641.639344:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298115840 00000100:00000001:0.0:1787828641.639346:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787828641.639347:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828641.639348:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787828641.639350:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828641/real 1787828641] req@ffff8d30c9e2cb40 x1874670646750336/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828666 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787828641.639355:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787828641.639356:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787828641.639357:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787828641.639358:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787828641.639359:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828641.639360:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828641.639361:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787828641.639364:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2cb40 x1874670646750336/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828666 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787828641.639369:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828641.639370:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787828641.639371:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828641.639372:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828641.639373:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828641.639374:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787828641.639375:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787828641.639376:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828641.639378:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787828641.639379:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787828641.639381:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646750336:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787828641.639384:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1787828641.639385:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787828641.639386:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828641.639388:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787828641.639390:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787828641.639391:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828641.639394:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2d2c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646750464:192.168.203.143@tcp:8:kworker.0 00000100:00000001:0.0:1787828641.639397:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787828641.639399:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787828641.639400:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828641.639402:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1787828641.639404:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828641.639409:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787828641.639411:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417873088 : -126234291678528 : ffff8d30c9e2d2c0) 00000100:00000001:0.0:1787828641.639414:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1787828641.639426:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:0.0:1787828641.639431:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1787828641.639433:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828641.639434:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828641.639436:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787828641.639439:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828641.639442:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298114595 00000100:00000001:0.0:1787828641.639444:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298115840 00000100:00000001:0.0:1787828641.639445:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298114595 00000100:00000001:0.0:1787828641.639447:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298115840 00000100:00000001:0.0:1787828641.639448:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787828641.639449:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828641.639451:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787828641.639453:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828641/real 1787828641] req@ffff8d30c9e2d2c0 x1874670646750464/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828666 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787828641.639459:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787828641.639459:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787828641.639461:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787828641.639462:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787828641.639463:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828641.639464:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828641.639465:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787828641.639467:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2d2c0 x1874670646750464/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828666 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787828641.639472:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828641.639473:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787828641.639474:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828641.639475:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828641.639476:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828641.639477:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787828641.639478:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787828641.639480:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828641.639482:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787828641.639483:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787828641.639485:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2d2c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646750464:192.168.203.143@tcp:8:kworker.0 00000100:00000001:0.0:1787828641.639488:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828641.639489:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828641.639490:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787828641.639491:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787828641.639492:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787828641.639493:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787828641.639494:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787828641.639495:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787828641.639496:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787828641.639499:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787828641.639499:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828641.639501:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828641.639501:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787828641.639502:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787828641.639503:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787828641.639504:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787828641.639505:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787828641.639506:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787828641.639507:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787828641.639508:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787828641.639509:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828641.639510:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828641.639511:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787828641.639511:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787828641.639512:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787828641.639513:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787828641.639514:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787828641.639514:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787828641.639515:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787828641.639516:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787828641.639517:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828641.639518:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828641.639520:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787828641.639521:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828641.639522:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787828641.639523:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828642.534710:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828642.534715:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828642.534778:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828642.534783:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828642.534787:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828643.558723:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828643.558728:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828643.558783:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828643.558787:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828643.558790:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828644.582680:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828644.582686:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828644.582738:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828644.582742:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828644.582744:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828645.606750:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828645.606757:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828645.606851:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828645.606858:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828645.606864:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828646.630730:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828646.630737:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828646.630814:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828646.630821:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828646.630827:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787828646.758737:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828646.758758:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1787828646.758762:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787828646.758766:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787828646.758771:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787828646.758779:0:468:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 3447 00000100:00080000:2.0:1787828646.758791:0:468:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787828646.758794:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828646.758797:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787828646.758802:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787828646.758805:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787828646.758819:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:2.0:1787828646.758823:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:2.0:1787828646.758826:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828646.758828:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:2.0:1787828646.758835:0:468:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787828646.758837:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828646.758841:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787828646.758844:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828646.758848:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828646.758852:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:2.0:1787828646.758855:0:468:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828646.758864:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787828646.758867:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828646.758871:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787828646.758886:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787828646.758890:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828646.758901:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787828646.758910:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2fc00 x1874670646750592/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787828646.758923:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787828646.758928:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828646.758947:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787828646.758951:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828646.758954:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1787828646.758956:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787828646.758958:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787828646.758964:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787828646.758968:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3447 00000100:00080000:2.0:1787828646.758974:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787828646.758979:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828646.758981:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787828646.758985:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787828646.758988:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787828646.758991:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:0.0:1787828646.758996:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:2.0:1787828646.758996:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:2.0:1787828646.759004:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787828646.759005:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1787828646.759007:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:2.0:1787828646.759016:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787828646.759024:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1787828646.759029:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828646.759030:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 02000000:00000001:2.0:1787828646.759038:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:0.0:1787828646.759040:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828646.759047:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787828646.759050:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787828646.759052:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828646.759057:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 02000000:00000001:0.0:1787828646.759058:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000008:00100000:2.0:1787828646.759062:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 02000000:00000001:0.0:1787828646.759067:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:2.0:1787828646.759068:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828646.759075:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828646.759091:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646750592:192.168.203.143@tcp:250:kworker.0 02000000:00000001:2.0:1787828646.759094:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787828646.759099:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828646.759101:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:2.0:1787828646.759103:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787828646.759107:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1787828646.759107:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787828646.759110:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828646.759112:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828646.759116:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:2.0:1787828646.759120:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828646.759123:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787828646.759133:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2de00 x1874670646750720/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787828646.759155:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787828646.759157:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787828646.759159:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828646.759164:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417883648 : -126234291667968 : ffff8d30c9e2fc00) 00000100:00000001:2.0:1787828646.759167:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828646.759174:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00080000:2.0:1787828646.759174:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828646.759182:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1787828646.759184:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787828646.759186:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787828646.759189:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787828646.759193:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3447 00000100:00080000:2.0:1787828646.759198:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787828646.759200:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828646.759201:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787828646.759203:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787828646.759205:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787828646.759207:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000020:00000001:2.0:1787828646.759210:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:2.0:1787828646.759213:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828646.759215:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:2.0:1787828646.759220:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787828646.759223:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828646.759225:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787828646.759228:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828646.759230:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828646.759234:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1787828646.759236:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1787828646.759239:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828646.759241:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828646.759247:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787828646.759250:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828646.759253:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787828646.759255:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787828646.759259:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:1.0:1787828646.759267:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:2.0:1787828646.759269:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787828646.759281:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2c3c0 x1874670646750848/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787828646.759284:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787828646.759293:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787828646.759307:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828646.759310:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787828646.759311:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1787828646.759320:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828646.759322:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828646.759325:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828646.759329:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4298120960 00000100:00080000:2.0:1787828646.759331:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828646.759338:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1787828646.759341:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:1.0:1787828646.759344:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 00000100:00080000:2.0:1787828646.759345:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787828646.759354:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:1.0:1787828646.759355:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4298120960 00000100:00080000:2.0:1787828646.759359:0:468:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3447 00000100:00000001:1.0:1787828646.759364:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 00000100:00080000:2.0:1787828646.759377:0:468:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1787828646.759379:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787828646.759383:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787828646.759385:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828646.759386:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:1.0:1787828646.759390:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000020:00000001:2.0:1787828646.759392:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787828646.759396:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00100000:1.0:1787828646.759401:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828646/real 1787828646] req@ffff8d30c9e2fc00 x1874670646750592/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828671 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:2.0:1787828646.759413:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000400:00000001:1.0:1787828646.759418:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000020:00000001:2.0:1787828646.759421:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000400:00000001:1.0:1787828646.759423:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1787828646.759440:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787828646.759441:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1787828646.759444:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:1.0:1787828646.759447:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787828646.759450:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828646.759454:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787828646.759457:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:2.0:1787828646.759461:0:468:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00100000:1.0:1787828646.759466:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2fc00 x1874670646750592/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828671 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787828646.759470:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828646.759479:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1787828646.759481:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828646.759486:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 02000000:00000001:2.0:1787828646.759487:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787828646.759492:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:2.0:1787828646.759493:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828646.759497:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787828646.759498:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828646.759506:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787828646.759509:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:1.0:1787828646.759511:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000008:00100000:2.0:1787828646.759513:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00080000:1.0:1787828646.759516:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000008:00000001:2.0:1787828646.759519:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828646.759524:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1787828646.759526:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787828646.759531:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 02000000:00000001:2.0:1787828646.759532:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787828646.759537:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787828646.759540:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646750592:192.168.203.143@tcp:250:kworker.0 00000100:00000001:2.0:1787828646.759542:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787828646.759549:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:1.0:1787828646.759551:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:2.0:1787828646.759553:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828646.759573:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787828646.759580:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 00000100:00000001:2.0:1787828646.759585:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828646.759588:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 00000100:00100000:2.0:1787828646.759596:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2c780 x1874670646750976/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:1.0:1787828646.759597:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00000001:2.0:1787828646.759646:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787828646.759650:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787828646.759655:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000100:00000001:2.0:1787828646.759658:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787828646.759661:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787828646.759670:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787828646.759674:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787828646.759683:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787828646.759686:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828646.759692:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828646.759702:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787828646.759705:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828646.759710:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787828646.759713:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1787828646.759715:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787828646.759719:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787828646.759722:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828646.759725:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787828646.759729:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787828646.759732:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787828646.759739:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646750720:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787828646.759744:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787828646.759747:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787828646.759750:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828646.759752:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787828646.759757:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828646.759767:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787828646.759769:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417875968 : -126234291675648 : ffff8d30c9e2de00) 00000100:00000001:1.0:1787828646.759776:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1787828646.759797:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787828646.759805:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787828646.759809:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787828646.759811:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828646.759813:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787828646.759818:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828646.759825:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298119703 00000100:00000001:1.0:1787828646.759841:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298120960 00000100:00000001:1.0:1787828646.759845:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298119703 00000100:00000001:1.0:1787828646.759848:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298120960 00000100:00000001:1.0:1787828646.759852:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787828646.759856:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828646.759859:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787828646.759866:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828646/real 1787828646] req@ffff8d30c9e2de00 x1874670646750720/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828671 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787828646.759879:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787828646.759881:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787828646.759886:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787828646.759889:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787828646.759891:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828646.759905:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787828646.759907:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787828646.759913:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2de00 x1874670646750720/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828671 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787828646.759925:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828646.759928:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787828646.759931:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828646.759933:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787828646.759934:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828646.759936:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787828646.759938:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787828646.759941:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828646.759945:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787828646.759947:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787828646.759950:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646750720:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787828646.759954:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787828646.759956:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787828646.759957:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828646.759959:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787828646.759961:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787828646.759963:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787828646.759966:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646750848:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787828646.759969:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787828646.759971:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787828646.759973:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828646.759974:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787828646.759978:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828646.759983:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787828646.759985:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417869248 : -126234291682368 : ffff8d30c9e2c3c0) 00000100:00000001:1.0:1787828646.759988:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1787828646.760002:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787828646.760006:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787828646.760009:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787828646.760010:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828646.760012:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787828646.760015:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828646.760019:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298119703 00000100:00000001:1.0:1787828646.760021:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298120960 00000100:00000001:1.0:1787828646.760023:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298119703 00000100:00000001:1.0:1787828646.760025:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298120960 00000100:00000001:1.0:1787828646.760027:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787828646.760029:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828646.760042:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787828646.760046:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828646/real 1787828646] req@ffff8d30c9e2c3c0 x1874670646750848/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828671 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787828646.760054:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787828646.760055:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787828646.760057:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787828646.760059:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787828646.760060:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828646.760062:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787828646.760063:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787828646.760067:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2c3c0 x1874670646750848/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828671 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787828646.760073:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828646.760075:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787828646.760077:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828646.760078:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787828646.760079:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828646.760081:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787828646.760082:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787828646.760085:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828646.760088:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787828646.760090:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787828646.760093:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646750848:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787828646.760096:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787828646.760097:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787828646.760098:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828646.760100:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787828646.760102:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787828646.760104:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787828646.760106:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646750976:192.168.203.143@tcp:8:kworker.0 00000100:00000001:1.0:1787828646.760112:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787828646.760114:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787828646.760115:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828646.760117:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787828646.760119:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828646.760124:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787828646.760125:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417870208 : -126234291681408 : ffff8d30c9e2c780) 00000100:00000001:1.0:1787828646.760128:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1787828646.760136:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787828646.760140:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787828646.760142:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787828646.760143:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828646.760145:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787828646.760147:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828646.760150:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298119716 00000100:00000001:1.0:1787828646.760152:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298120960 00000100:00000001:1.0:1787828646.760154:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298119716 00000100:00000001:1.0:1787828646.760156:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298120960 00000100:00000001:1.0:1787828646.760158:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787828646.760160:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828646.760161:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787828646.760164:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828646/real 1787828646] req@ffff8d30c9e2c780 x1874670646750976/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828671 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787828646.760171:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787828646.760172:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787828646.760174:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787828646.760175:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787828646.760176:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828646.760178:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787828646.760179:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787828646.760182:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2c780 x1874670646750976/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828671 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787828646.760195:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828646.760197:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787828646.760199:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828646.760200:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787828646.760202:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828646.760203:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787828646.760204:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787828646.760207:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828646.760209:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787828646.760211:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787828646.760214:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646750976:192.168.203.143@tcp:8:kworker.0 00000100:00000001:1.0:1787828646.760217:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828646.760219:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787828646.760220:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787828646.760221:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787828646.760224:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787828646.760225:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787828646.760226:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787828646.760229:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787828646.760230:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787828646.760234:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787828646.760236:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828646.760237:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787828646.760238:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787828646.760239:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787828646.760241:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787828646.760242:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787828646.760243:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787828646.760244:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787828646.760245:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787828646.760248:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787828646.760249:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828646.760250:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787828646.760251:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787828646.760253:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787828646.760255:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787828646.760256:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787828646.760257:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787828646.760259:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787828646.760260:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787828646.760261:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787828646.760262:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828646.760264:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828646.760267:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787828646.760269:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828646.760271:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787828646.760272:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828647.654726:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828647.654732:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828647.654753:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828647.654757:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828647.654759:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828648.678667:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828648.678672:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828648.678725:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828648.678729:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828648.678731:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828649.702712:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828649.702718:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828649.702769:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828649.702773:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828649.702776:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828650.726709:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828650.726717:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828650.726816:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828650.726821:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828650.726825:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828651.750701:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828651.750717:0:815:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1787828651.750719:0:815:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1787828651.750722:0:815:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828651.750729:0:815:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1787828651.750731:0:815:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828651.750733:0:815:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828651.750735:0:815:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828651.750804:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828651.750808:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828651.750812:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787828651.878717:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828651.878728:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1787828651.878730:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787828651.878734:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787828651.878739:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787828651.878747:0:40701:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 3452 00000100:00080000:3.0:1787828651.878757:0:40701:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787828651.878761:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787828651.878763:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:3.0:1787828651.878767:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1787828651.878770:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1787828651.878773:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:3.0:1787828651.878776:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:3.0:1787828651.878779:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787828651.878780:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:3.0:1787828651.878786:0:40701:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:3.0:1787828651.878789:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787828651.878800:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787828651.878803:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787828651.878807:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828651.878809:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:3.0:1787828651.878812:0:40701:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787828651.878822:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787828651.878825:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828651.878828:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787828651.878830:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787828651.878835:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828651.878846:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787828651.878856:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c7016d00 x1874670646751104/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787828651.878868:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787828651.878872:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828651.878885:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787828651.878889:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828651.878892:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1787828651.878894:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787828651.878896:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787828651.878901:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787828651.878904:0:40701:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3452 00000100:00000001:1.0:1787828651.878907:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00080000:3.0:1787828651.878909:0:40701:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1787828651.878912:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787828651.878913:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787828651.878915:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787828651.878917:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:1.0:1787828651.878918:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787828651.878921:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828651.878923:0:40701:0:(genops.c:792:class_conn2export()) Process entered 02000000:00000001:1.0:1787828651.878927:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000020:00000001:3.0:1787828651.878927:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 02000000:00000001:1.0:1787828651.878930:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000020:00000001:3.0:1787828651.878932:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 02000000:00000001:1.0:1787828651.878934:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828651.878938:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00100000:1.0:1787828651.878942:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c7016d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646751104:192.168.203.143@tcp:250:kworker.0 00000100:00000001:3.0:1787828651.878944:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787828651.878949:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:3.0:1787828651.878950:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 02000000:00000001:1.0:1787828651.878954:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787828651.878958:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787828651.878958:0:40701:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 02000000:00000001:1.0:1787828651.878961:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:3.0:1787828651.878965:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828651.878967:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787828651.878968:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787828651.878976:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787828651.878979:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828651.878982:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787828651.878982:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:1.0:1787828651.878986:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369547008 : -126234340004608 : ffff8d30c7016d00) 00000100:00000001:1.0:1787828651.878990:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000008:00100000:3.0:1787828651.878990:0:40701:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1787828651.878996:0:40701:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828651.878999:0:40701:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787828651.879005:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787828651.879007:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828651.879009:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787828651.879010:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787828651.879013:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828651.879018:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:1.0:1787828651.879024:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00100000:3.0:1787828651.879027:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c70170c0 x1874670646751232/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787828651.879035:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787828651.879040:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787828651.879042:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787828651.879045:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828651.879046:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828651.879050:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1787828651.879053:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828651.879055:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787828651.879059:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1787828651.879060:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4298126080 00000100:00080000:3.0:1787828651.879067:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1787828651.879070:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:1.0:1787828651.879071:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 00000100:00080000:3.0:1787828651.879073:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787828651.879078:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787828651.879081:0:40701:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3452 00000100:00000001:1.0:1787828651.879085:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4298126080 00000100:00080000:3.0:1787828651.879086:0:40701:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787828651.879091:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787828651.879092:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 00000100:00000001:3.0:1787828651.879096:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:1.0:1787828651.879097:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787828651.879103:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828651.879103:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:1.0:1787828651.879108:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000020:00000001:3.0:1787828651.879108:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1787828651.879113:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00100000:1.0:1787828651.879117:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828651/real 1787828651] req@ffff8d30c7016d00 x1874670646751104/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828676 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:3.0:1787828651.879119:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:3.0:1787828651.879122:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787828651.879123:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:3.0:1787828651.879127:0:40701:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:3.0:1787828651.879134:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787828651.879138:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000400:00000001:1.0:1787828651.879141:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787828651.879144:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 02000000:00000001:3.0:1787828651.879144:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787828651.879149:0:2390:0:(events.c:78:reply_in_callback()) Process entered 02000000:00000001:3.0:1787828651.879150:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828651.879152:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787828651.879155:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787828651.879155:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:1.0:1787828651.879160:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000008:00100000:3.0:1787828651.879161:0:40701:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00000001:1.0:1787828651.879162:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787828651.879169:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c7016d00 x1874670646751104/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828676 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00000001:3.0:1787828651.879169:0:40701:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828651.879174:0:40701:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828651.879182:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:3.0:1787828651.879183:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:1.0:1787828651.879185:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 02000000:00000001:3.0:1787828651.879187:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828651.879188:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828651.879193:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787828651.879193:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:1.0:1787828651.879196:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828651.879198:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787828651.879200:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787828651.879202:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828651.879207:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 02000000:00000001:3.0:1787828651.879207:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:1.0:1787828651.879212:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 02000000:00000001:3.0:1787828651.879213:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787828651.879221:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c7016d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646751104:192.168.203.143@tcp:250:kworker.0 00000100:00000001:3.0:1787828651.879222:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828651.879232:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787828651.879235:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c7017840 x1874670646751360/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787828651.879236:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787828651.879239:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787828651.879242:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787828651.879247:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00000001:3.0:1787828651.879249:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000020:00000001:1.0:1787828651.879251:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000100:00000001:3.0:1787828651.879254:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787828651.879256:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 00000100:00000001:3.0:1787828651.879260:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828651.879262:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 00000100:00080000:3.0:1787828651.879265:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 02000000:00000001:1.0:1787828651.879266:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787828651.879270:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787828651.879273:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:3.0:1787828651.879273:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1787828651.879278:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:1.0:1787828651.879279:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:3.0:1787828651.879283:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787828651.879287:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787828651.879288:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:1.0:1787828651.879290:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828651.879294:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00080000:3.0:1787828651.879295:0:40701:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3452 00000100:00000001:1.0:1787828651.879297:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1787828651.879299:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787828651.879301:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787828651.879303:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787828651.879303:0:40701:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 02000000:00000001:1.0:1787828651.879305:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:3.0:1787828651.879309:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 02000000:00000001:1.0:1787828651.879310:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787828651.879314:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 02000000:00000001:1.0:1787828651.879315:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828651.879318:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1787828651.879323:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00100000:1.0:1787828651.879324:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c70170c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646751232:192.168.203.143@tcp:38:kworker.0 00000020:00000001:3.0:1787828651.879326:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000020:00000001:3.0:1787828651.879332:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:1.0:1787828651.879334:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:3.0:1787828651.879338:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 02000000:00000001:1.0:1787828651.879339:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:3.0:1787828651.879342:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 02000000:00000001:1.0:1787828651.879343:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828651.879348:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00080000:3.0:1787828651.879353:0:40701:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 02000000:00000001:1.0:1787828651.879354:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828651.879359:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787828651.879363:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1787828651.879364:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787828651.879366:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369547968 : -126234340003648 : ffff8d30c70170c0) 02000000:00000001:3.0:1787828651.879368:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787828651.879370:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 02000000:00000001:3.0:1787828651.879373:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828651.879377:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:3.0:1787828651.879380:0:40701:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1787828651.879384:0:40701:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:1.0:1787828651.879386:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:3.0:1787828651.879388:0:40701:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828651.879391:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787828651.879394:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787828651.879396:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787828651.879396:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787828651.879400:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828651.879401:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787828651.879404:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828651.879406:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:1.0:1787828651.879409:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298124822 02000000:00000001:3.0:1787828651.879409:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:1.0:1787828651.879414:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298126080 02000000:00000001:3.0:1787828651.879414:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828651.879419:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298124822 00000100:00000001:1.0:1787828651.879421:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298126080 00000100:00000001:3.0:1787828651.879423:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828651.879424:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787828651.879429:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828651.879431:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787828651.879436:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828651/real 1787828651] req@ffff8d30c70170c0 x1874670646751232/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828676 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787828651.879436:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c7014000 x1874670646751488/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000001:1.0:1787828651.879465:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787828651.879468:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787828651.879468:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787828651.879470:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787828651.879473:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787828651.879474:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828651.879475:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828651.879480:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787828651.879482:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787828651.879483:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787828651.879486:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c70170c0 x1874670646751232/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828676 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787828651.879493:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828651.879495:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787828651.879497:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828651.879498:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787828651.879498:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828651.879500:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787828651.879501:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787828651.879503:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828651.879505:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787828651.879507:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787828651.879510:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c70170c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646751232:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787828651.879512:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787828651.879513:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787828651.879515:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828651.879516:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787828651.879532:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787828651.879535:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787828651.879538:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c7017840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646751360:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787828651.879540:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787828651.879541:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787828651.879542:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828651.879544:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787828651.879546:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828651.879550:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787828651.879551:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369549888 : -126234340001728 : ffff8d30c7017840) 00000100:00000001:1.0:1787828651.879554:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1787828651.879562:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787828651.879565:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787828651.879567:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787828651.879568:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828651.879569:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787828651.879571:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828651.879573:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298124822 00000100:00000001:1.0:1787828651.879575:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298126080 00000100:00000001:1.0:1787828651.879577:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298124822 00000100:00000001:1.0:1787828651.879578:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298126080 00000100:00000001:1.0:1787828651.879579:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787828651.879581:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828651.879582:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787828651.879584:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828651/real 1787828651] req@ffff8d30c7017840 x1874670646751360/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828676 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787828651.879589:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787828651.879590:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787828651.879592:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787828651.879593:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787828651.879594:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828651.879595:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787828651.879596:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787828651.879598:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c7017840 x1874670646751360/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828676 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787828651.879625:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828651.879626:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787828651.879628:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828651.879629:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787828651.879629:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828651.879631:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787828651.879632:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787828651.879633:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828651.879635:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787828651.879637:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787828651.879639:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c7017840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646751360:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787828651.879642:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828651.879643:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787828651.879644:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787828651.879645:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787828651.879647:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787828651.879648:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787828651.879649:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787828651.879650:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787828651.879651:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787828651.879652:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787828651.879653:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828651.879654:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787828651.879655:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787828651.879656:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787828651.879657:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787828651.879658:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787828651.879659:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787828651.879660:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787828651.879660:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787828651.879661:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787828651.879662:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828651.879663:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828651.879665:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787828651.879667:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828651.879668:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787828651.879669:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1787828651.879670:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787828651.879671:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787828651.879672:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828651.879673:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787828651.879675:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787828651.879676:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787828651.879678:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c7014000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646751488:192.168.203.143@tcp:8:kworker.0 00000100:00000001:1.0:1787828651.879680:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787828651.879681:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787828651.879682:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828651.879684:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787828651.879685:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828651.879689:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787828651.879690:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369535488 : -126234340016128 : ffff8d30c7014000) 00000100:00000001:1.0:1787828651.879693:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1787828651.879700:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787828651.879703:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787828651.879705:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787828651.879706:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828651.879707:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787828651.879708:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828651.879710:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298124836 00000100:00000001:1.0:1787828651.879712:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298126080 00000100:00000001:1.0:1787828651.879714:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298124836 00000100:00000001:1.0:1787828651.879715:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298126080 00000100:00000001:1.0:1787828651.879716:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787828651.879718:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828651.879719:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787828651.879721:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828651/real 1787828651] req@ffff8d30c7014000 x1874670646751488/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828676 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787828651.879731:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787828651.879731:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787828651.879733:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787828651.879734:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787828651.879736:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828651.879737:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787828651.879739:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787828651.879743:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c7014000 x1874670646751488/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828676 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787828651.879750:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828651.879752:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787828651.879754:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828651.879756:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787828651.879757:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828651.879760:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787828651.879761:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787828651.879764:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828651.879768:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787828651.879770:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787828651.879774:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c7014000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646751488:192.168.203.143@tcp:8:kworker.0 00000100:00000001:1.0:1787828651.879778:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828651.879780:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787828651.879782:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787828651.879783:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787828651.879786:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787828651.879787:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787828651.879788:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787828651.879790:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787828651.879791:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787828651.879793:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787828651.879793:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828651.879795:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828651.879797:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787828651.879798:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828651.879800:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787828651.879801:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828652.774663:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828652.774669:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828652.774729:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828652.774734:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828652.774738:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828653.798767:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828653.798774:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828653.798834:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828653.798840:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828653.798844:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828654.822776:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828654.822783:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828654.822917:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828654.822924:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828654.822930:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828655.846670:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828655.846675:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828655.846730:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828655.846735:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828655.846739:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828656.870713:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828656.870718:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828656.870806:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828656.870814:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828656.870819:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787828656.998668:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828656.998679:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1787828656.998682:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787828656.998686:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787828656.998691:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787828656.998707:0:40701:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 3457 00000100:00080000:3.0:1787828656.998718:0:40701:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787828656.998721:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787828656.998723:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:3.0:1787828656.998727:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1787828656.998730:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1787828656.998733:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:3.0:1787828656.998735:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:3.0:1787828656.998737:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787828656.998738:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:3.0:1787828656.998742:0:40701:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:3.0:1787828656.998743:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787828656.998745:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787828656.998749:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787828656.998751:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828656.998753:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:3.0:1787828656.998755:0:40701:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787828656.998768:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787828656.998770:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828656.998773:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787828656.998776:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787828656.998780:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828656.998793:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787828656.998803:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c7017c00 x1874670646751616/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787828656.998812:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787828656.998817:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828656.998831:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787828656.998835:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828656.998839:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1787828656.998841:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787828656.998842:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787828656.998846:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787828656.998849:0:40701:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3457 00000100:00080000:3.0:1787828656.998853:0:40701:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787828656.998855:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787828656.998857:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:3.0:1787828656.998859:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1787828656.998860:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1787828656.998863:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000020:00000001:3.0:1787828656.998865:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:3.0:1787828656.998868:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787828656.998869:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:1.0:1787828656.998871:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00080000:3.0:1787828656.998873:0:40701:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:3.0:1787828656.998875:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828656.998877:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 02000000:00000001:3.0:1787828656.998877:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1787828656.998880:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787828656.998883:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 02000000:00000001:3.0:1787828656.998883:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787828656.998885:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828656.998886:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828656.998888:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 02000000:00000001:1.0:1787828656.998890:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787828656.998893:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000008:00100000:3.0:1787828656.998893:0:40701:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 02000000:00000001:1.0:1787828656.998897:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:3.0:1787828656.998897:0:40701:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828656.998899:0:40701:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787828656.998902:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c7017c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646751616:192.168.203.143@tcp:250:kworker.0 02000000:00000001:3.0:1787828656.998903:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787828656.998905:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828656.998906:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:1.0:1787828656.998907:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1787828656.998908:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787828656.998910:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828656.998913:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:3.0:1787828656.998914:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828656.998917:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787828656.998919:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c7016940 x1874670646751744/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:1.0:1787828656.998920:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787828656.998925:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828656.998941:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787828656.998943:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787828656.998943:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828656.998945:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369550848 : -126234340000768 : ffff8d30c7017c00) 00000100:00000001:3.0:1787828656.998946:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828656.998949:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00080000:3.0:1787828656.998949:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828656.998952:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1787828656.998954:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787828656.998955:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787828656.998958:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787828656.998960:0:40701:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3457 00000100:00080000:3.0:1787828656.998963:0:40701:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787828656.998965:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787828656.998967:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:3.0:1787828656.998969:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1787828656.998970:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1787828656.998972:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000020:00000001:3.0:1787828656.998974:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:3.0:1787828656.998976:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787828656.998977:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:3.0:1787828656.998980:0:40701:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:3.0:1787828656.998983:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787828656.998984:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787828656.998987:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787828656.998988:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828656.998990:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:3.0:1787828656.998992:0:40701:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1787828656.998995:0:40701:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:1.0:1787828656.998996:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:3.0:1787828656.998997:0:40701:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787828656.999002:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787828656.999004:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828656.999006:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787828656.999007:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787828656.999009:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828656.999012:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787828656.999013:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828656.999016:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00100000:3.0:1787828656.999018:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c7014000 x1874670646751872/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787828656.999019:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828656.999021:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1787828656.999025:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787828656.999027:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828656.999028:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828656.999029:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787828656.999032:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1787828656.999033:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4298131200 00000100:00080000:3.0:1787828656.999035:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1787828656.999037:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787828656.999038:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787828656.999040:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787828656.999043:0:40701:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3457 00000100:00000001:1.0:1787828656.999044:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 00000100:00080000:3.0:1787828656.999046:0:40701:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787828656.999048:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787828656.999050:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:3.0:1787828656.999051:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1787828656.999053:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1787828656.999055:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000020:00000001:3.0:1787828656.999057:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:3.0:1787828656.999060:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787828656.999062:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4298131200 00000100:00000001:3.0:1787828656.999062:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:3.0:1787828656.999065:0:40701:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:1.0:1787828656.999067:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 00000100:00000001:3.0:1787828656.999068:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787828656.999069:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1787828656.999071:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787828656.999072:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787828656.999074:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787828656.999074:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828656.999076:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:1.0:1787828656.999077:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000008:00100000:3.0:1787828656.999078:0:40701:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1787828656.999081:0:40701:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787828656.999083:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828656/real 1787828656] req@ffff8d30c7017c00 x1874670646751616/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828681 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787828656.999083:0:40701:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787828656.999088:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787828656.999089:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828656.999092:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787828656.999093:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787828656.999095:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:1.0:1787828656.999096:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787828656.999098:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787828656.999100:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828656.999103:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00100000:3.0:1787828656.999105:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c7017840 x1874670646752000/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787828656.999106:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787828656.999108:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787828656.999112:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787828656.999114:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828656.999116:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828656.999119:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787828656.999122:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787828656.999126:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c7017c00 x1874670646751616/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828681 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787828656.999134:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828656.999138:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787828656.999141:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828656.999143:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787828656.999145:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828656.999148:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787828656.999150:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787828656.999153:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828656.999159:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787828656.999161:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787828656.999165:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c7017c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646751616:192.168.203.143@tcp:250:kworker.0 00000100:00000001:1.0:1787828656.999169:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828656.999171:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787828656.999172:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787828656.999175:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787828656.999178:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787828656.999180:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787828656.999182:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787828656.999187:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787828656.999188:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787828656.999190:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787828656.999191:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828656.999193:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828656.999201:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787828656.999204:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828656.999205:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787828656.999206:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1787828656.999207:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787828656.999209:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787828656.999210:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828656.999211:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787828656.999213:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787828656.999214:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787828656.999217:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c7016940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646751744:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787828656.999219:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787828656.999221:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787828656.999222:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828656.999223:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787828656.999225:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828656.999231:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787828656.999232:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369546048 : -126234340005568 : ffff8d30c7016940) 00000100:00000001:1.0:1787828656.999236:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1787828656.999245:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787828656.999248:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787828656.999250:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787828656.999251:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828656.999252:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787828656.999254:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828656.999258:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298129942 00000100:00000001:1.0:1787828656.999261:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298131200 00000100:00000001:1.0:1787828656.999262:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298129942 00000100:00000001:1.0:1787828656.999264:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298131200 00000100:00000001:1.0:1787828656.999265:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787828656.999267:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828656.999268:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787828656.999270:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828656/real 1787828656] req@ffff8d30c7016940 x1874670646751744/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828681 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787828656.999281:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787828656.999282:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787828656.999283:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787828656.999284:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787828656.999285:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828656.999286:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787828656.999288:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787828656.999290:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c7016940 x1874670646751744/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828681 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787828656.999295:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828656.999296:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787828656.999297:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828656.999298:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787828656.999299:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828656.999300:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787828656.999301:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787828656.999303:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828656.999304:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787828656.999306:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787828656.999308:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c7016940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646751744:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787828656.999310:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787828656.999310:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787828656.999312:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828656.999313:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787828656.999314:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787828656.999316:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787828656.999317:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c7014000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646751872:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787828656.999319:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787828656.999320:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787828656.999322:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828656.999323:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787828656.999325:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828656.999328:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787828656.999329:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369535488 : -126234340016128 : ffff8d30c7014000) 00000100:00000001:1.0:1787828656.999331:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1787828656.999337:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787828656.999339:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787828656.999341:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787828656.999342:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828656.999343:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787828656.999344:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828656.999346:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298129942 00000100:00000001:1.0:1787828656.999348:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298131200 00000100:00000001:1.0:1787828656.999350:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298129942 00000100:00000001:1.0:1787828656.999351:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298131200 00000100:00000001:1.0:1787828656.999352:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787828656.999353:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828656.999354:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787828656.999357:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828656/real 1787828656] req@ffff8d30c7014000 x1874670646751872/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828681 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787828656.999361:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787828656.999362:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787828656.999363:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787828656.999364:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787828656.999366:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828656.999367:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787828656.999368:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787828656.999370:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c7014000 x1874670646751872/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828681 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787828656.999374:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828656.999375:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787828656.999377:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828656.999378:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787828656.999379:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828656.999380:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787828656.999381:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787828656.999383:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828656.999385:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787828656.999386:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787828656.999388:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c7014000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646751872:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787828656.999390:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787828656.999391:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787828656.999391:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828656.999393:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787828656.999394:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787828656.999395:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787828656.999396:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c7017840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646752000:192.168.203.143@tcp:8:kworker.0 00000100:00000001:1.0:1787828656.999398:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787828656.999399:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787828656.999400:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828656.999401:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787828656.999403:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828656.999406:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787828656.999406:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369549888 : -126234340001728 : ffff8d30c7017840) 00000100:00000001:1.0:1787828656.999409:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1787828656.999413:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787828656.999416:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787828656.999417:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787828656.999418:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828656.999419:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787828656.999421:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828656.999423:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298129955 00000100:00000001:1.0:1787828656.999425:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298131200 00000100:00000001:1.0:1787828656.999427:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298129955 00000100:00000001:1.0:1787828656.999428:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298131200 00000100:00000001:1.0:1787828656.999430:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787828656.999431:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828656.999432:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787828656.999435:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828656/real 1787828656] req@ffff8d30c7017840 x1874670646752000/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828681 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787828656.999439:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787828656.999440:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787828656.999441:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787828656.999442:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787828656.999443:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828656.999443:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787828656.999444:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787828656.999446:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c7017840 x1874670646752000/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828681 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787828656.999450:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828656.999451:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787828656.999453:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828656.999454:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787828656.999455:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828656.999456:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787828656.999457:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787828656.999459:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828656.999461:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787828656.999462:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787828656.999464:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c7017840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646752000:192.168.203.143@tcp:8:kworker.0 00000100:00000001:1.0:1787828656.999466:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828656.999467:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787828656.999468:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787828656.999469:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787828656.999471:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787828656.999472:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787828656.999473:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787828656.999475:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787828656.999475:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787828656.999477:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787828656.999478:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828656.999479:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787828656.999480:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787828656.999481:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787828656.999498:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787828656.999499:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787828656.999500:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787828656.999501:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787828656.999502:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787828656.999503:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787828656.999504:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828656.999506:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787828656.999511:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787828656.999512:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787828656.999514:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787828656.999514:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787828656.999515:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787828656.999519:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787828656.999520:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787828656.999521:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787828656.999522:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828656.999523:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828656.999525:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787828656.999527:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828656.999528:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787828656.999529:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828657.894720:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828657.894725:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828657.894783:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828657.894788:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828657.894791:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828658.918709:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828658.918714:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828658.918834:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828658.918840:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828658.918843:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828659.942750:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828659.942758:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828659.942833:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828659.942839:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828659.942843:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828660.966716:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828660.966723:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828660.966788:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828660.966794:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828660.966799:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828661.990774:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828661.990783:0:815:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1787828661.990785:0:815:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1787828661.990788:0:815:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828661.990792:0:815:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1787828661.990795:0:815:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828661.990797:0:815:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828661.990801:0:815:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828661.990887:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828661.990893:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828661.990897:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787828662.118725:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828662.118736:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1787828662.118738:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787828662.118743:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787828662.118747:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787828662.118753:0:468:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 3462 00000100:00080000:2.0:1787828662.118762:0:468:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787828662.118764:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828662.118766:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787828662.118769:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787828662.118772:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787828662.118774:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:2.0:1787828662.118777:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:2.0:1787828662.118779:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828662.118781:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:2.0:1787828662.118786:0:468:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787828662.118789:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828662.118792:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787828662.118796:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828662.118800:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828662.118803:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:2.0:1787828662.118806:0:468:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828662.118817:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787828662.118819:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828662.118822:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787828662.118825:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787828662.118829:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828662.118839:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787828662.118848:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2e940 x1874670646752128/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787828662.118856:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787828662.118859:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828662.118872:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787828662.118874:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828662.118876:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1787828662.118877:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787828662.118879:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787828662.118881:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787828662.118882:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3462 00000100:00080000:2.0:1787828662.118885:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787828662.118886:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828662.118887:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787828662.118889:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787828662.118890:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787828662.118891:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000020:00000001:2.0:1787828662.118893:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:2.0:1787828662.118896:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828662.118897:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:2.0:1787828662.118902:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787828662.118905:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828662.118908:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1787828662.118912:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:2.0:1787828662.118914:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787828662.118918:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 02000000:00000001:2.0:1787828662.118919:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828662.118921:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1787828662.118923:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:1.0:1787828662.118925:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787828662.118927:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000008:00100000:2.0:1787828662.118930:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 02000000:00000001:1.0:1787828662.118932:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787828662.118936:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000008:00000001:2.0:1787828662.118936:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828662.118941:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828662.118942:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787828662.118950:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646752128:192.168.203.143@tcp:250:kworker.0 02000000:00000001:2.0:1787828662.118950:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787828662.118953:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828662.118957:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:1.0:1787828662.118958:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787828662.118959:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787828662.118962:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828662.118963:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787828662.118966:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828662.118967:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828662.118969:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787828662.118976:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787828662.118976:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2f0c0 x1874670646752256/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787828662.118984:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787828662.118986:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828662.118989:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828662.118990:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00080000:2.0:1787828662.118991:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1787828662.118994:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417878848 : -126234291672768 : ffff8d30c9e2e940) 00000100:00080000:2.0:1787828662.118995:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1787828662.118998:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787828662.119000:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787828662.119003:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:2.0:1787828662.119003:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787828662.119007:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3462 00000100:00080000:2.0:1787828662.119011:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787828662.119012:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828662.119013:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787828662.119015:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787828662.119016:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787828662.119018:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000020:00000001:2.0:1787828662.119020:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:2.0:1787828662.119023:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828662.119024:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:2.0:1787828662.119028:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787828662.119031:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828662.119033:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787828662.119036:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828662.119039:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828662.119041:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000800:00000100:1.0:1787828662.119045:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000008:00100000:2.0:1787828662.119046:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1787828662.119051:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828662.119053:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828662.119058:0:2390:0:(events.c:34:request_out_callback()) Process entered 02000000:00000001:2.0:1787828662.119059:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787828662.119062:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828662.119064:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828662.119065:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:1.0:1787828662.119067:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828662.119068:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:1.0:1787828662.119070:0:2390:0:(events.c:68:request_out_callback()) Process leaving 02000000:00000001:2.0:1787828662.119072:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828662.119077:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828662.119078:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828662.119082:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4298136320 00000100:00100000:2.0:1787828662.119086:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2f480 x1874670646752384/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787828662.119094:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 00000100:00000001:2.0:1787828662.119094:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787828662.119098:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828662.119103:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787828662.119105:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828662.119109:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1787828662.119110:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4298136320 00000100:00000001:2.0:1787828662.119111:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787828662.119116:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787828662.119117:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 00000100:00000001:1.0:1787828662.119122:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787828662.119123:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:1.0:1787828662.119124:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828662.119127:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00080000:2.0:1787828662.119129:0:468:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3462 00000100:00100000:1.0:1787828662.119134:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828662/real 1787828662] req@ffff8d30c9e2e940 x1874670646752128/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828687 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828662.119137:0:468:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787828662.119143:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828662.119146:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787828662.119150:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787828662.119153:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000400:00000001:1.0:1787828662.119156:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000020:00000001:2.0:1787828662.119156:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000400:00000001:1.0:1787828662.119162:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000020:00000001:2.0:1787828662.119162:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:1.0:1787828662.119169:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787828662.119171:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787828662.119173:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787828662.119176:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828662.119176:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:1.0:1787828662.119183:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00080000:2.0:1787828662.119184:0:468:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:1.0:1787828662.119188:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828662.119189:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828662.119194:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787828662.119197:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00100000:1.0:1787828662.119199:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2e940 x1874670646752128/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828687 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1787828662.119200:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828662.119205:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1787828662.119208:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000008:00000001:2.0:1787828662.119213:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828662.119217:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828662.119223:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:1.0:1787828662.119224:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:2.0:1787828662.119226:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828662.119229:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787828662.119229:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787828662.119233:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:1.0:1787828662.119234:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828662.119237:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 02000000:00000001:2.0:1787828662.119238:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828662.119239:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828662.119244:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787828662.119249:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828662.119253:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787828662.119259:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:2.0:1787828662.119260:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2c000 x1874670646752512/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787828662.119264:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787828662.119268:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787828662.119272:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646752128:192.168.203.143@tcp:250:kworker.0 00000100:00000001:1.0:1787828662.119276:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828662.119278:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787828662.119279:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787828662.119281:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787828662.119285:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00000001:2.0:1787828662.119286:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000020:00000001:1.0:1787828662.119288:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000100:00000001:2.0:1787828662.119289:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787828662.119291:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 00000100:00000001:2.0:1787828662.119294:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828662.119295:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787828662.119298:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787828662.119302:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787828662.119303:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828662.119305:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828662.119311:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787828662.119313:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828662.119315:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787828662.119316:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1787828662.119317:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787828662.119319:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787828662.119320:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828662.119321:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787828662.119322:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787828662.119324:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787828662.119327:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646752256:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787828662.119329:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787828662.119330:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787828662.119332:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828662.119333:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787828662.119335:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828662.119340:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787828662.119341:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417880768 : -126234291670848 : ffff8d30c9e2f0c0) 00000100:00000001:1.0:1787828662.119344:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1787828662.119355:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787828662.119358:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787828662.119359:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787828662.119360:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828662.119361:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787828662.119364:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828662.119367:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298135062 00000100:00000001:1.0:1787828662.119369:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298136320 00000100:00000001:1.0:1787828662.119370:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298135062 00000100:00000001:1.0:1787828662.119372:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298136320 00000100:00000001:1.0:1787828662.119373:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787828662.119375:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828662.119376:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787828662.119378:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828662/real 1787828662] req@ffff8d30c9e2f0c0 x1874670646752256/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828687 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787828662.119384:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787828662.119385:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787828662.119386:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787828662.119387:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787828662.119388:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828662.119389:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787828662.119390:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787828662.119393:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2f0c0 x1874670646752256/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828687 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787828662.119402:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828662.119403:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787828662.119405:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828662.119406:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787828662.119407:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828662.119408:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787828662.119409:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787828662.119411:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828662.119412:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787828662.119414:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787828662.119416:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646752256:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787828662.119418:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787828662.119419:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787828662.119420:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828662.119421:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787828662.119422:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787828662.119423:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787828662.119425:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646752384:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787828662.119427:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787828662.119428:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787828662.119430:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828662.119431:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787828662.119432:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828662.119435:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787828662.119436:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417881728 : -126234291669888 : ffff8d30c9e2f480) 00000100:00000001:1.0:1787828662.119438:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1787828662.119444:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787828662.119462:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787828662.119464:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787828662.119465:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828662.119467:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787828662.119470:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828662.119472:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298135062 00000100:00000001:1.0:1787828662.119474:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298136320 00000100:00000001:1.0:1787828662.119476:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298135062 00000100:00000001:1.0:1787828662.119477:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298136320 00000100:00000001:1.0:1787828662.119479:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787828662.119480:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828662.119481:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787828662.119484:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828662/real 1787828662] req@ffff8d30c9e2f480 x1874670646752384/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828687 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787828662.119489:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787828662.119490:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787828662.119491:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787828662.119492:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787828662.119493:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828662.119494:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787828662.119495:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787828662.119498:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2f480 x1874670646752384/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828687 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787828662.119502:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828662.119504:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787828662.119505:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828662.119506:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787828662.119507:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828662.119508:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787828662.119509:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787828662.119511:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828662.119512:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787828662.119514:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787828662.119516:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646752384:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787828662.119519:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787828662.119520:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787828662.119521:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828662.119522:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787828662.119523:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787828662.119524:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787828662.119526:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646752512:192.168.203.143@tcp:8:kworker.0 00000100:00000001:1.0:1787828662.119528:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787828662.119529:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787828662.119531:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828662.119532:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787828662.119534:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828662.119537:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787828662.119538:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417868288 : -126234291683328 : ffff8d30c9e2c000) 00000100:00000001:1.0:1787828662.119540:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1787828662.119548:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787828662.119551:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787828662.119552:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787828662.119553:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828662.119554:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787828662.119556:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828662.119558:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298135075 00000100:00000001:1.0:1787828662.119559:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298136320 00000100:00000001:1.0:1787828662.119561:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298135075 00000100:00000001:1.0:1787828662.119562:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298136320 00000100:00000001:1.0:1787828662.119563:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787828662.119565:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828662.119566:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787828662.119568:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828662/real 1787828662] req@ffff8d30c9e2c000 x1874670646752512/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828687 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787828662.119577:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787828662.119578:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787828662.119579:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787828662.119580:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787828662.119581:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828662.119582:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787828662.119583:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787828662.119585:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2c000 x1874670646752512/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828687 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787828662.119589:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828662.119590:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787828662.119591:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828662.119592:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787828662.119593:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828662.119594:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787828662.119596:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787828662.119597:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828662.119636:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787828662.119639:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787828662.119641:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646752512:192.168.203.143@tcp:8:kworker.0 00000100:00000001:1.0:1787828662.119644:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828662.119645:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787828662.119646:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787828662.119647:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787828662.119649:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787828662.119650:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787828662.119651:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787828662.119653:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787828662.119654:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787828662.119656:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787828662.119657:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828662.119659:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787828662.119659:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787828662.119660:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787828662.119661:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787828662.119662:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787828662.119663:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787828662.119664:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787828662.119665:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787828662.119666:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787828662.119666:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828662.119667:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787828662.119668:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787828662.119669:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787828662.119670:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787828662.119670:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787828662.119671:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787828662.119672:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787828662.119673:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787828662.119674:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787828662.119675:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828662.119676:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828662.119678:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787828662.119679:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828662.119680:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787828662.119681:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828663.014694:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828663.014699:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828663.014767:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828663.014772:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828663.014776:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828664.038700:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828664.038706:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828664.038756:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828664.038760:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828664.038763:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828665.062737:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828665.062755:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828665.062821:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828665.062825:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828665.062830:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828666.086695:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828666.086701:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828666.086751:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828666.086754:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828666.086756:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828667.110697:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828667.110702:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828667.110770:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828667.110775:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828667.110779:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787828667.238693:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828667.238702:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1787828667.238705:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787828667.238707:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787828667.238711:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787828667.238717:0:468:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 3467 00000100:00080000:0.0:1787828667.238726:0:468:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787828667.238728:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787828667.238729:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787828667.238732:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787828667.238734:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787828667.238737:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:0.0:1787828667.238739:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:0.0:1787828667.238740:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787828667.238741:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:0.0:1787828667.238744:0:468:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787828667.238747:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828667.238749:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787828667.238751:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787828667.238753:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828667.238755:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1787828667.238757:0:468:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828667.238767:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787828667.238769:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828667.238772:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787828667.238775:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787828667.238779:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828667.238791:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828667.238801:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d270c0 x1874670646752640/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787828667.238821:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787828667.238825:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828667.238837:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787828667.238840:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828667.238843:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1787828667.238845:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787828667.238847:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787828667.238850:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787828667.238854:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3467 00000100:00080000:0.0:1787828667.238858:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787828667.238862:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787828667.238864:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787828667.238867:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:1.0:1787828667.238868:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:0.0:1787828667.238869:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787828667.238874:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:1.0:1787828667.238874:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000020:00000001:0.0:1787828667.238879:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:1.0:1787828667.238879:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787828667.238882:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787828667.238884:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787828667.238885:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828667.238888:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 02000000:00000001:1.0:1787828667.238890:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000100:00080000:0.0:1787828667.238893:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 02000000:00000001:1.0:1787828667.238894:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787828667.238899:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828667.238900:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828667.238904:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00100000:1.0:1787828667.238908:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d270c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646752640:192.168.203.143@tcp:250:kworker.0 02000000:00000001:0.0:1787828667.238910:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787828667.238914:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828667.238914:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:0.0:1787828667.238917:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 02000000:00000001:1.0:1787828667.238919:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000008:00100000:0.0:1787828667.238921:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 02000000:00000001:1.0:1787828667.238922:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828667.238927:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000008:00000001:0.0:1787828667.238928:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828667.238932:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828667.238933:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828667.238940:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787828667.238942:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828667.238944:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787828667.238946:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:1.0:1787828667.238947:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 02000000:00000001:0.0:1787828667.238948:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828667.238950:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601348800 : -126234108202816 : ffff8d30d4d270c0) 00000100:00000001:1.0:1787828667.238954:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:0.0:1787828667.238958:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828667.238966:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d24000 x1874670646752768/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787828667.238974:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787828667.238975:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828667.238979:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787828667.238981:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828667.238991:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1787828667.238994:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000800:00000100:1.0:1787828667.238994:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00080000:0.0:1787828667.238997:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787828667.239001:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787828667.239003:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3467 00000100:00080000:0.0:1787828667.239007:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787828667.239009:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787828667.239010:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:1.0:1787828667.239010:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000020:00000001:0.0:1787828667.239014:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787828667.239015:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:1.0:1787828667.239017:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000020:00000001:0.0:1787828667.239018:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000020:00000001:0.0:1787828667.239021:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:1.0:1787828667.239022:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828667.239023:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787828667.239025:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:1.0:1787828667.239029:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00080000:0.0:1787828667.239030:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787828667.239033:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828667.239034:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787828667.239037:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787828667.239038:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828667.239039:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828667.239040:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1787828667.239043:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1787828667.239046:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828667.239047:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828667.239047:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4298141440 02000000:00000001:0.0:1787828667.239052:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787828667.239054:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828667.239056:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787828667.239057:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787828667.239058:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828667.239060:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1787828667.239063:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828667.239068:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d24f00 x1874670646752896/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787828667.239069:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4298141440 00000100:00000001:0.0:1787828667.239077:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787828667.239077:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1787828667.239081:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828667.239082:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787828667.239084:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828667.239087:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787828667.239088:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828667.239093:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1787828667.239093:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00000001:0.0:1787828667.239096:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787828667.239098:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787828667.239100:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00100000:1.0:1787828667.239102:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828667/real 1787828667] req@ffff8d30d4d270c0 x1874670646752640/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828692 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828667.239104:0:468:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3467 00000100:00080000:0.0:1787828667.239109:0:468:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787828667.239112:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787828667.239115:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787828667.239120:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787828667.239121:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787828667.239122:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000400:00000001:1.0:1787828667.239122:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000020:00000001:0.0:1787828667.239126:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000400:00000001:1.0:1787828667.239126:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787828667.239130:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787828667.239131:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:1.0:1787828667.239132:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00080000:0.0:1787828667.239135:0:468:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:1.0:1787828667.239137:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787828667.239138:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828667.239140:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:0.0:1787828667.239141:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1787828667.239144:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 02000000:00000001:0.0:1787828667.239145:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787828667.239146:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:0.0:1787828667.239148:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828667.239151:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1787828667.239152:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1787828667.239153:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828667.239154:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828667.239158:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787828667.239161:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787828667.239162:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d270c0 x1874670646752640/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828692 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787828667.239165:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787828667.239168:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787828667.239171:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828667.239175:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828667.239179:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828667.239180:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787828667.239184:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828667.239186:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787828667.239188:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787828667.239190:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d25a40 x1874670646753024/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787828667.239190:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787828667.239193:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787828667.239199:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828667.239207:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787828667.239226:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787828667.239227:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000001:0.0:1787828667.239230:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787828667.239235:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d270c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646752640:192.168.203.143@tcp:250:kworker.0 00000100:00000001:0.0:1787828667.239236:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828667.239245:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828667.239248:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787828667.239249:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787828667.239251:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787828667.239254:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787828667.239256:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787828667.239258:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787828667.239261:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787828667.239262:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787828667.239266:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787828667.239278:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828667.239280:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828667.239287:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787828667.239288:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828667.239290:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787828667.239291:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1787828667.239292:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787828667.239294:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787828667.239295:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828667.239296:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787828667.239298:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787828667.239300:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787828667.239302:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d24000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646752768:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787828667.239305:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787828667.239306:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787828667.239308:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828667.239309:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787828667.239311:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828667.239316:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787828667.239317:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601336320 : -126234108215296 : ffff8d30d4d24000) 00000100:00000001:1.0:1787828667.239321:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1787828667.239333:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787828667.239336:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787828667.239338:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787828667.239339:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828667.239341:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787828667.239343:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828667.239346:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298140182 00000100:00000001:1.0:1787828667.239349:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298141440 00000100:00000001:1.0:1787828667.239351:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298140182 00000100:00000001:1.0:1787828667.239352:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298141440 00000100:00000001:1.0:1787828667.239354:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787828667.239355:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828667.239357:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787828667.239360:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828667/real 1787828667] req@ffff8d30d4d24000 x1874670646752768/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828692 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787828667.239366:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787828667.239367:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787828667.239368:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787828667.239370:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787828667.239371:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828667.239372:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787828667.239373:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787828667.239376:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d24000 x1874670646752768/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828692 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787828667.239382:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828667.239383:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787828667.239385:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828667.239386:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787828667.239387:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828667.239388:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787828667.239389:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787828667.239391:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828667.239394:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787828667.239396:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787828667.239399:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d24000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646752768:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787828667.239401:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787828667.239402:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787828667.239403:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828667.239405:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787828667.239406:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787828667.239407:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787828667.239409:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d24f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646752896:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787828667.239432:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787828667.239445:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787828667.239447:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828667.239448:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787828667.239450:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828667.239470:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787828667.239471:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601340160 : -126234108211456 : ffff8d30d4d24f00) 00000100:00000001:1.0:1787828667.239473:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1787828667.239482:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787828667.239485:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787828667.239486:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787828667.239487:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828667.239488:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787828667.239490:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828667.239493:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298140182 00000100:00000001:1.0:1787828667.239494:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298141440 00000100:00000001:1.0:1787828667.239496:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298140182 00000100:00000001:1.0:1787828667.239497:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298141440 00000100:00000001:1.0:1787828667.239510:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787828667.239512:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828667.239513:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787828667.239516:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828667/real 1787828667] req@ffff8d30d4d24f00 x1874670646752896/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828692 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787828667.239521:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787828667.239522:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787828667.239524:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787828667.239525:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787828667.239526:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828667.239527:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787828667.239528:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787828667.239530:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d24f00 x1874670646752896/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828692 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787828667.239534:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828667.239536:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787828667.239537:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828667.239538:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787828667.239539:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828667.239540:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787828667.239541:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787828667.239542:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828667.239544:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787828667.239546:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787828667.239548:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d24f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646752896:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787828667.239551:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787828667.239551:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787828667.239552:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828667.239554:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787828667.239555:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787828667.239556:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787828667.239558:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d25a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646753024:192.168.203.143@tcp:8:kworker.0 00000100:00000001:1.0:1787828667.239559:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787828667.239561:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787828667.239562:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828667.239563:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787828667.239566:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828667.239569:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787828667.239570:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601343040 : -126234108208576 : ffff8d30d4d25a40) 00000100:00000001:1.0:1787828667.239572:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1787828667.239578:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787828667.239580:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787828667.239581:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787828667.239582:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828667.239583:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787828667.239585:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828667.239587:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298140195 00000100:00000001:1.0:1787828667.239588:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298141440 00000100:00000001:1.0:1787828667.239590:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298140195 00000100:00000001:1.0:1787828667.239591:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298141440 00000100:00000001:1.0:1787828667.239592:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787828667.239594:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828667.239595:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787828667.239597:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828667/real 1787828667] req@ffff8d30d4d25a40 x1874670646753024/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828692 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787828667.239601:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787828667.239602:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787828667.239603:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787828667.239604:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787828667.239605:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828667.239606:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787828667.239607:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787828667.239609:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d25a40 x1874670646753024/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828692 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787828667.239646:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828667.239647:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787828667.239649:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828667.239650:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787828667.239651:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828667.239652:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787828667.239653:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787828667.239655:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828667.239657:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787828667.239659:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787828667.239661:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d25a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646753024:192.168.203.143@tcp:8:kworker.0 00000100:00000001:1.0:1787828667.239663:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828667.239664:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787828667.239665:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787828667.239666:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787828667.239668:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787828667.239669:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787828667.239670:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787828667.239671:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787828667.239672:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787828667.239674:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787828667.239675:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828667.239676:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787828667.239677:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787828667.239678:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787828667.239679:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787828667.239680:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787828667.239681:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787828667.239682:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787828667.239682:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787828667.239684:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787828667.239685:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828667.239686:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787828667.239686:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787828667.239687:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787828667.239688:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787828667.239689:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787828667.239690:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787828667.239691:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787828667.239692:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787828667.239694:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787828667.239694:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828667.239696:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828667.239698:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787828667.239699:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828667.239700:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787828667.239701:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828668.134690:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828668.134695:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828668.134752:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828668.134758:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828668.134761:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828669.158695:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828669.158701:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828669.158761:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828669.158767:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828669.158771:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828670.182685:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828670.182690:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828670.182734:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828670.182737:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828670.182740:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828671.206672:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828671.206678:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828671.206734:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828671.206738:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828671.206742:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828672.230685:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828672.230692:0:815:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1787828672.230694:0:815:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1787828672.230696:0:815:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828672.230700:0:815:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1787828672.230702:0:815:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828672.230704:0:815:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828672.230706:0:815:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828672.230766:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828672.230770:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828672.230778:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787828672.358693:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828672.358701:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1787828672.358703:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787828672.358706:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787828672.358710:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787828672.358716:0:468:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 3472 00000100:00080000:0.0:1787828672.358725:0:468:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787828672.358727:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787828672.358728:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787828672.358732:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787828672.358734:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787828672.358737:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:0.0:1787828672.358738:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:0.0:1787828672.358740:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787828672.358740:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:0.0:1787828672.358744:0:468:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787828672.358746:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828672.358748:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787828672.358751:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787828672.358753:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828672.358754:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1787828672.358756:0:468:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828672.358766:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787828672.358767:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828672.358769:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787828672.358772:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787828672.358784:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828672.358798:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828672.358805:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d25a40 x1874670646753152/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787828672.358813:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787828672.358817:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828672.358830:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787828672.358833:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828672.358835:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1787828672.358836:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787828672.358838:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787828672.358840:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787828672.358841:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3472 00000100:00080000:0.0:1787828672.358844:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787828672.358845:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787828672.358846:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787828672.358848:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787828672.358849:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787828672.358850:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000020:00000001:0.0:1787828672.358852:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:0.0:1787828672.358853:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787828672.358854:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:0.0:1787828672.358857:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787828672.358860:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828672.358861:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787828672.358866:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787828672.358867:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828672.358868:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787828672.358869:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:1.0:1787828672.358874:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000008:00100000:0.0:1787828672.358875:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00000001:1.0:1787828672.358877:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000008:00000001:0.0:1787828672.358878:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828672.358881:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787828672.358882:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828672.358884:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828672.358886:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787828672.358889:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828672.358890:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:0.0:1787828672.358891:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1787828672.358893:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787828672.358894:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787828672.358897:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828672.358898:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828672.358903:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787828672.358905:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d25a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646753152:192.168.203.143@tcp:250:kworker.0 00000100:00100000:0.0:1787828672.358910:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d24f00 x1874670646753280/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787828672.358911:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787828672.358916:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787828672.358919:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828672.358920:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787828672.358923:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828672.358923:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:0.0:1787828672.358927:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828672.358928:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787828672.358930:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828672.358934:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1787828672.358935:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787828672.358937:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787828672.358939:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787828672.358940:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3472 00000100:00080000:0.0:1787828672.358943:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787828672.358944:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787828672.358945:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:1.0:1787828672.358945:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000020:00000001:0.0:1787828672.358946:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:1.0:1787828672.358948:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601343040 : -126234108208576 : ffff8d30d4d25a40) 00000020:00000001:0.0:1787828672.358949:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787828672.358952:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:1.0:1787828672.358954:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000020:00000001:0.0:1787828672.358956:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:0.0:1787828672.358959:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787828672.358959:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:0.0:1787828672.358962:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787828672.358964:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828672.358965:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787828672.358967:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787828672.358969:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828672.358970:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1787828672.358971:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1787828672.358973:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828672.358974:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828672.358977:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787828672.358978:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828672.358979:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787828672.358980:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787828672.358982:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828672.358986:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:1.0:1787828672.358989:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00100000:0.0:1787828672.358993:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d24000 x1874670646753408/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787828672.359000:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787828672.359005:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828672.359006:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787828672.359008:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828672.359010:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828672.359011:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787828672.359014:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787828672.359017:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1787828672.359017:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787828672.359020:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1787828672.359023:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:1.0:1787828672.359023:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4298146560 00000100:00080000:0.0:1787828672.359026:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787828672.359029:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787828672.359031:0:468:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3472 00000100:00000001:1.0:1787828672.359033:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 00000100:00080000:0.0:1787828672.359034:0:468:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787828672.359038:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787828672.359039:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787828672.359040:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787828672.359042:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:1.0:1787828672.359042:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4298146560 00000020:00000001:0.0:1787828672.359043:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000020:00000001:0.0:1787828672.359048:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:1.0:1787828672.359049:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1787828672.359050:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787828672.359053:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:1.0:1787828672.359054:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00080000:0.0:1787828672.359057:0:468:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:1.0:1787828672.359059:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828672.359061:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828672.359062:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 02000000:00000001:0.0:1787828672.359064:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787828672.359067:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00100000:1.0:1787828672.359069:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828672/real 1787828672] req@ffff8d30d4d25a40 x1874670646753152/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828697 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:0.0:1787828672.359070:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828672.359074:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1787828672.359077:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1787828672.359081:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828672.359086:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828672.359090:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787828672.359091:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828672.359093:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787828672.359093:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787828672.359094:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828672.359097:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:1.0:1787828672.359097:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787828672.359100:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00100000:0.0:1787828672.359102:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d270c0 x1874670646753536/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787828672.359104:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787828672.359108:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787828672.359110:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828672.359111:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787828672.359114:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828672.359114:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787828672.359117:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828672.359119:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787828672.359124:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d25a40 x1874670646753152/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828697 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787828672.359131:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828672.359134:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787828672.359137:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828672.359138:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787828672.359139:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828672.359141:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787828672.359142:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787828672.359144:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828672.359147:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787828672.359149:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787828672.359151:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d25a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646753152:192.168.203.143@tcp:250:kworker.0 00000100:00000001:1.0:1787828672.359155:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828672.359156:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787828672.359158:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787828672.359159:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787828672.359161:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787828672.359163:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787828672.359164:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787828672.359172:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787828672.359174:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787828672.359177:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787828672.359178:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828672.359179:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828672.359185:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787828672.359186:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828672.359188:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787828672.359189:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1787828672.359190:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787828672.359191:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787828672.359192:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828672.359193:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787828672.359195:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787828672.359196:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787828672.359198:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d24f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646753280:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787828672.359201:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787828672.359202:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787828672.359203:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828672.359204:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787828672.359206:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828672.359210:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787828672.359211:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601340160 : -126234108211456 : ffff8d30d4d24f00) 00000100:00000001:1.0:1787828672.359214:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1787828672.359223:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787828672.359226:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787828672.359227:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787828672.359228:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828672.359229:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787828672.359231:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828672.359234:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298145302 00000100:00000001:1.0:1787828672.359236:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298146560 00000100:00000001:1.0:1787828672.359238:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298145302 00000100:00000001:1.0:1787828672.359239:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298146560 00000100:00000001:1.0:1787828672.359241:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787828672.359242:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828672.359244:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787828672.359246:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828672/real 1787828672] req@ffff8d30d4d24f00 x1874670646753280/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828697 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787828672.359256:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787828672.359257:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787828672.359259:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787828672.359261:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787828672.359262:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828672.359264:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787828672.359265:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787828672.359269:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d24f00 x1874670646753280/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828697 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787828672.359276:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828672.359278:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787828672.359280:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828672.359281:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787828672.359282:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828672.359284:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787828672.359286:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787828672.359289:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828672.359292:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787828672.359294:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787828672.359296:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d24f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646753280:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787828672.359300:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787828672.359301:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787828672.359302:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828672.359304:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787828672.359306:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787828672.359308:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787828672.359311:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d24000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646753408:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787828672.359317:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787828672.359319:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787828672.359320:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828672.359322:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787828672.359325:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828672.359330:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787828672.359331:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601336320 : -126234108215296 : ffff8d30d4d24000) 00000100:00000001:1.0:1787828672.359334:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1787828672.359346:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787828672.359351:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787828672.359353:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787828672.359354:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828672.359356:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787828672.359358:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828672.359362:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298145302 00000100:00000001:1.0:1787828672.359365:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298146560 00000100:00000001:1.0:1787828672.359367:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298145302 00000100:00000001:1.0:1787828672.359369:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298146560 00000100:00000001:1.0:1787828672.359371:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787828672.359373:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828672.359375:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787828672.359391:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828672/real 1787828672] req@ffff8d30d4d24000 x1874670646753408/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828697 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787828672.359399:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787828672.359400:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787828672.359403:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787828672.359405:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787828672.359406:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828672.359408:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787828672.359409:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787828672.359413:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d24000 x1874670646753408/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828697 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787828672.359426:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828672.359429:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787828672.359431:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828672.359432:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787828672.359434:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828672.359436:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787828672.359437:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787828672.359440:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828672.359443:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787828672.359445:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787828672.359448:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d24000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646753408:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787828672.359452:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787828672.359453:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787828672.359454:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828672.359456:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787828672.359458:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787828672.359460:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787828672.359463:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d270c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646753536:192.168.203.143@tcp:8:kworker.0 00000100:00000001:1.0:1787828672.359466:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787828672.359468:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787828672.359470:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828672.359472:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787828672.359475:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828672.359480:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787828672.359482:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601348800 : -126234108202816 : ffff8d30d4d270c0) 00000100:00000001:1.0:1787828672.359485:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1787828672.359494:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787828672.359498:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787828672.359500:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787828672.359501:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828672.359502:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787828672.359504:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828672.359506:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298145315 00000100:00000001:1.0:1787828672.359508:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298146560 00000100:00000001:1.0:1787828672.359509:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298145315 00000100:00000001:1.0:1787828672.359511:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298146560 00000100:00000001:1.0:1787828672.359512:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787828672.359513:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828672.359515:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787828672.359517:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828672/real 1787828672] req@ffff8d30d4d270c0 x1874670646753536/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828697 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787828672.359522:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787828672.359523:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787828672.359524:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787828672.359525:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787828672.359526:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828672.359527:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787828672.359528:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787828672.359531:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d270c0 x1874670646753536/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828697 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787828672.359535:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828672.359537:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787828672.359538:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828672.359539:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787828672.359540:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828672.359541:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787828672.359542:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787828672.359544:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828672.359545:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787828672.359547:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787828672.359549:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d270c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646753536:192.168.203.143@tcp:8:kworker.0 00000100:00000001:1.0:1787828672.359552:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828672.359553:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787828672.359554:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787828672.359555:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787828672.359556:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787828672.359557:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787828672.359558:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787828672.359560:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787828672.359560:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787828672.359563:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787828672.359564:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828672.359565:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787828672.359566:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787828672.359566:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787828672.359568:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787828672.359569:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787828672.359569:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787828672.359570:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787828672.359571:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787828672.359572:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787828672.359573:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828672.359574:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787828672.359574:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787828672.359575:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787828672.359584:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787828672.359585:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787828672.359585:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787828672.359586:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787828672.359587:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787828672.359590:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787828672.359590:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828672.359592:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828672.359594:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787828672.359595:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828672.359596:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787828672.359597:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828673.254659:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828673.254664:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828673.254705:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828673.254708:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828673.254711:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828674.278721:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828674.278727:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828674.278796:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828674.278799:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828674.278802:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828675.302660:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828675.302664:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828675.302737:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828675.302740:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828675.302743:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828676.326713:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828676.326719:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828676.326795:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828676.326799:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828676.326802:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828677.350668:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828677.350674:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828677.350721:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828677.350725:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828677.350727:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787828677.478736:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828677.478750:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1787828677.478752:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787828677.478755:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787828677.478759:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787828677.478766:0:40701:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 3477 00000100:00080000:3.0:1787828677.478776:0:40701:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787828677.478778:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787828677.478783:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:3.0:1787828677.478786:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1787828677.478789:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1787828677.478791:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:3.0:1787828677.478793:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:3.0:1787828677.478795:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787828677.478796:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:3.0:1787828677.478801:0:40701:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:3.0:1787828677.478804:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787828677.478807:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787828677.478810:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787828677.478813:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828677.478815:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:3.0:1787828677.478818:0:40701:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787828677.478830:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787828677.478832:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828677.478836:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787828677.478839:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787828677.478843:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828677.478855:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787828677.478863:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c70170c0 x1874670646753664/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787828677.478871:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787828677.478875:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828677.478887:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787828677.478889:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828677.478891:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1787828677.478892:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787828677.478894:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787828677.478896:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787828677.478898:0:40701:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3477 00000100:00080000:3.0:1787828677.478901:0:40701:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787828677.478903:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787828677.478904:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:3.0:1787828677.478906:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1787828677.478907:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1787828677.478910:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000020:00000001:3.0:1787828677.478911:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:3.0:1787828677.478913:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787828677.478914:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:3.0:1787828677.478917:0:40701:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:1.0:1787828677.478919:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1787828677.478919:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787828677.478923:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1787828677.478925:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1787828677.478929:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 02000000:00000001:3.0:1787828677.478929:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787828677.478933:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 02000000:00000001:3.0:1787828677.478934:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828677.478937:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828677.478937:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:3.0:1787828677.478943:0:40701:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 02000000:00000001:1.0:1787828677.478944:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000008:00000001:3.0:1787828677.478948:0:40701:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828677.478949:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787828677.478953:0:40701:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828677.478954:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787828677.478958:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00100000:1.0:1787828677.478960:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c70170c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646753664:192.168.203.143@tcp:250:kworker.0 02000000:00000001:3.0:1787828677.478961:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828677.478965:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787828677.478967:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:1.0:1787828677.478969:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1787828677.478970:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828677.478976:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:3.0:1787828677.478977:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828677.478982:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787828677.478984:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c7016d00 x1874670646753792/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:1.0:1787828677.478987:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:3.0:1787828677.478993:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 02000000:00000001:1.0:1787828677.478995:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828677.478996:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828677.479002:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787828677.479004:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828677.479006:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1787828677.479007:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787828677.479009:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787828677.479010:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787828677.479012:0:40701:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3477 00000100:00080000:3.0:1787828677.479014:0:40701:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1787828677.479016:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787828677.479017:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787828677.479018:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:1.0:1787828677.479021:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369547968 : -126234340003648 : ffff8d30c70170c0) 00000020:00000001:3.0:1787828677.479021:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1787828677.479024:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:1.0:1787828677.479026:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000020:00000001:3.0:1787828677.479027:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000020:00000001:3.0:1787828677.479031:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:3.0:1787828677.479032:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787828677.479034:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:3.0:1787828677.479036:0:40701:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:3.0:1787828677.479037:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787828677.479039:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787828677.479040:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787828677.479042:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828677.479043:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:3.0:1787828677.479044:0:40701:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1787828677.479046:0:40701:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828677.479047:0:40701:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787828677.479050:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787828677.479051:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828677.479059:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787828677.479061:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787828677.479062:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828677.479066:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:1.0:1787828677.479071:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00100000:3.0:1787828677.479073:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c7014b40 x1874670646753920/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787828677.479079:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787828677.479081:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828677.479083:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828677.479085:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00080000:3.0:1787828677.479085:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828677.479090:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1787828677.479091:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787828677.479092:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:1.0:1787828677.479095:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787828677.479095:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787828677.479098:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:1.0:1787828677.479099:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00080000:3.0:1787828677.479102:0:40701:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3477 00000100:00000001:1.0:1787828677.479106:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787828677.479106:0:40701:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787828677.479110:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787828677.479112:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:1.0:1787828677.479113:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4298151680 00000020:00000001:3.0:1787828677.479114:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1787828677.479116:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1787828677.479119:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000020:00000001:3.0:1787828677.479121:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:1.0:1787828677.479123:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 00000100:00000001:3.0:1787828677.479123:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787828677.479124:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:1.0:1787828677.479130:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4298151680 00000100:00080000:3.0:1787828677.479130:0:40701:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:3.0:1787828677.479132:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787828677.479135:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787828677.479137:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787828677.479138:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828677.479139:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:3.0:1787828677.479140:0:40701:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1787828677.479142:0:40701:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828677.479145:0:40701:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828677.479148:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 02000000:00000001:3.0:1787828677.479148:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787828677.479151:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828677.479154:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787828677.479155:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:1.0:1787828677.479156:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787828677.479156:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828677.479162:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828677.479163:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787828677.479166:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c70152c0 x1874670646754048/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787828677.479168:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00000001:3.0:1787828677.479175:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787828677.479178:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787828677.479180:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828677/real 1787828677] req@ffff8d30c70170c0 x1874670646753664/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828702 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787828677.479181:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:1.0:1787828677.479193:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787828677.479195:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787828677.479199:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787828677.479201:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787828677.479203:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828677.479205:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787828677.479207:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787828677.479210:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c70170c0 x1874670646753664/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828702 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787828677.479216:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828677.479218:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787828677.479220:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828677.479221:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787828677.479222:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828677.479224:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787828677.479225:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787828677.479227:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828677.479229:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787828677.479231:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787828677.479233:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c70170c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646753664:192.168.203.143@tcp:250:kworker.0 00000100:00000001:1.0:1787828677.479237:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828677.479238:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787828677.479239:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787828677.479241:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787828677.479244:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787828677.479245:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787828677.479247:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787828677.479250:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787828677.479251:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787828677.479253:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787828677.479254:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828677.479256:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828677.479262:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787828677.479263:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828677.479264:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787828677.479266:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1787828677.479267:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787828677.479268:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787828677.479269:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828677.479271:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787828677.479273:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787828677.479274:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787828677.479277:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c7016d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646753792:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787828677.479279:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787828677.479280:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787828677.479281:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828677.479282:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787828677.479284:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828677.479289:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787828677.479290:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369547008 : -126234340004608 : ffff8d30c7016d00) 00000100:00000001:1.0:1787828677.479293:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1787828677.479303:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787828677.479306:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787828677.479307:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787828677.479308:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828677.479309:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787828677.479311:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828677.479314:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298150422 00000100:00000001:1.0:1787828677.479317:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298151680 00000100:00000001:1.0:1787828677.479318:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298150422 00000100:00000001:1.0:1787828677.479320:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298151680 00000100:00000001:1.0:1787828677.479321:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787828677.479322:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828677.479324:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787828677.479326:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828677/real 1787828677] req@ffff8d30c7016d00 x1874670646753792/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828702 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787828677.479331:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787828677.479332:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787828677.479334:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787828677.479335:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787828677.479335:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828677.479337:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787828677.479337:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787828677.479357:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c7016d00 x1874670646753792/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828702 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787828677.479363:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828677.479364:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787828677.479366:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828677.479367:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787828677.479368:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828677.479369:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787828677.479370:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787828677.479371:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828677.479374:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787828677.479375:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787828677.479378:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c7016d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646753792:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787828677.479380:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787828677.479381:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787828677.479382:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828677.479384:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787828677.479385:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787828677.479386:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787828677.479388:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c7014b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646753920:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787828677.479390:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787828677.479391:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787828677.479392:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828677.479394:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787828677.479396:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828677.479399:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787828677.479400:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369538368 : -126234340013248 : ffff8d30c7014b40) 00000100:00000001:1.0:1787828677.479403:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1787828677.479411:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787828677.479414:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787828677.479416:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787828677.479417:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828677.479418:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787828677.479420:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828677.479422:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298150422 00000100:00000001:1.0:1787828677.479424:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298151680 00000100:00000001:1.0:1787828677.479425:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298150422 00000100:00000001:1.0:1787828677.479427:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298151680 00000100:00000001:1.0:1787828677.479428:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787828677.479429:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828677.479431:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787828677.479433:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828677/real 1787828677] req@ffff8d30c7014b40 x1874670646753920/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828702 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787828677.479438:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787828677.479439:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787828677.479440:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787828677.479441:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787828677.479442:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828677.479443:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787828677.479444:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787828677.479447:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c7014b40 x1874670646753920/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828702 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787828677.479450:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828677.479452:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787828677.479453:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828677.479454:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787828677.479455:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828677.479456:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787828677.479457:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787828677.479458:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828677.479460:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787828677.479461:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787828677.479463:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c7014b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646753920:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787828677.479466:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787828677.479467:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787828677.479468:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828677.479469:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787828677.479470:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787828677.479471:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787828677.479473:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c70152c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646754048:192.168.203.143@tcp:8:kworker.0 00000100:00000001:1.0:1787828677.479475:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787828677.479476:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787828677.479477:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828677.479478:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787828677.479479:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828677.479482:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787828677.479483:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369540288 : -126234340011328 : ffff8d30c70152c0) 00000100:00000001:1.0:1787828677.479485:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1787828677.479490:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787828677.479501:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787828677.479502:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787828677.479503:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828677.479504:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787828677.479506:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828677.479508:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298150435 00000100:00000001:1.0:1787828677.479510:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298151680 00000100:00000001:1.0:1787828677.479511:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298150435 00000100:00000001:1.0:1787828677.479513:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298151680 00000100:00000001:1.0:1787828677.479514:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787828677.479515:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828677.479516:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787828677.479519:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828677/real 1787828677] req@ffff8d30c70152c0 x1874670646754048/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828702 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787828677.479528:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787828677.479529:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787828677.479530:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787828677.479531:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787828677.479532:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828677.479533:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787828677.479534:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787828677.479536:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c70152c0 x1874670646754048/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828702 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787828677.479541:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828677.479542:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787828677.479543:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828677.479544:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787828677.479545:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828677.479546:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787828677.479547:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787828677.479549:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828677.479550:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787828677.479552:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787828677.479553:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c70152c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646754048:192.168.203.143@tcp:8:kworker.0 00000100:00000001:1.0:1787828677.479555:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828677.479557:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787828677.479557:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787828677.479558:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787828677.479560:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787828677.479561:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787828677.479561:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787828677.479563:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787828677.479564:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787828677.479565:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787828677.479566:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828677.479567:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787828677.479568:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787828677.479569:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787828677.479570:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787828677.479571:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787828677.479572:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787828677.479573:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787828677.479574:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787828677.479575:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787828677.479575:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828677.479576:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787828677.479577:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787828677.479578:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787828677.479578:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787828677.479579:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787828677.479580:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787828677.479581:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787828677.479581:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787828677.479582:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787828677.479583:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828677.479584:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828677.479586:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787828677.479587:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828677.479588:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787828677.479589:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828678.374690:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828678.374698:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828678.374753:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828678.374756:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828678.374759:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828679.398664:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828679.398667:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828679.398744:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828679.398750:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828679.398754:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828680.422684:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828680.422689:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828680.422767:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828680.422772:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828680.422777:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828681.446741:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828681.446751:0:815:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1787828681.446771:0:815:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1787828681.446776:0:815:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828681.446781:0:815:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1787828681.446785:0:815:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828681.446788:0:815:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828681.446792:0:815:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828681.446863:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828681.446867:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828681.446871:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828682.470707:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828682.470712:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828682.470772:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828682.470777:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828682.470780:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787828682.598667:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828682.598676:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1787828682.598678:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787828682.598681:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787828682.598685:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787828682.598691:0:40701:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 3483 00000100:00080000:3.0:1787828682.598701:0:40701:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787828682.598704:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787828682.598706:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:3.0:1787828682.598709:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1787828682.598711:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1787828682.598714:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:3.0:1787828682.598715:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:3.0:1787828682.598717:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787828682.598718:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:3.0:1787828682.598722:0:40701:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:3.0:1787828682.598724:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787828682.598726:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787828682.598729:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787828682.598732:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828682.598734:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:3.0:1787828682.598736:0:40701:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787828682.598744:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787828682.598745:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828682.598748:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787828682.598750:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787828682.598753:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828682.598762:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787828682.598769:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c7015680 x1874670646754176/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787828682.598777:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787828682.598781:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828682.598796:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787828682.598799:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828682.598802:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1787828682.598803:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787828682.598805:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787828682.598807:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787828682.598810:0:40701:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3483 00000100:00080000:3.0:1787828682.598814:0:40701:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787828682.598817:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787828682.598818:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:3.0:1787828682.598821:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1787828682.598823:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1787828682.598826:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000020:00000001:3.0:1787828682.598829:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:3.0:1787828682.598831:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787828682.598833:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:3.0:1787828682.598838:0:40701:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:3.0:1787828682.598842:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787828682.598845:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:2.0:1787828682.598846:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:3.0:1787828682.598851:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787828682.598853:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 02000000:00000001:3.0:1787828682.598856:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828682.598858:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787828682.598861:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:2.0:1787828682.598862:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787828682.598866:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000008:00100000:3.0:1787828682.598867:0:40701:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1787828682.598871:0:40701:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828682.598872:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:3.0:1787828682.598873:0:40701:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828682.598878:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787828682.598879:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787828682.598882:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828682.598886:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828682.598886:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787828682.598888:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787828682.598892:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787828682.598896:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c7015680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646754176:192.168.203.143@tcp:250:kworker.0 00000100:00000001:3.0:1787828682.598899:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828682.598901:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787828682.598906:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000100:00100000:3.0:1787828682.598907:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c70143c0 x1874670646754304/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:2.0:1787828682.598909:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828682.598915:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:3.0:1787828682.598919:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 02000000:00000001:2.0:1787828682.598921:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828682.598935:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787828682.598936:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369541248 : -126234340010368 : ffff8d30c7015680) 00000100:00000001:2.0:1787828682.598940:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1787828682.598971:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:2.0:1787828682.598980:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1787828682.598983:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828682.598985:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828682.598986:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1787828682.598990:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828682.598994:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4298156800 00000100:00000001:2.0:1787828682.599000:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 00000100:00000001:2.0:1787828682.599005:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4298156800 00000100:00000001:2.0:1787828682.599009:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 00000100:00000001:3.0:1787828682.599011:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828682.599012:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787828682.599016:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828682.599019:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00000001:3.0:1787828682.599023:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787828682.599025:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828682/real 1787828682] req@ffff8d30c7015680 x1874670646754176/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828707 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828682.599027:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828682.599032:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1787828682.599035:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787828682.599036:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000400:00000001:2.0:1787828682.599047:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787828682.599049:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1787828682.599053:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1787828682.599055:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787828682.599057:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828682.599060:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828682.599062:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787828682.599068:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c7015680 x1874670646754176/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828707 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787828682.599076:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828682.599079:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787828682.599082:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828682.599083:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828682.599084:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828682.599086:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787828682.599087:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787828682.599089:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828682.599093:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:2.0:1787828682.599095:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1787828682.599098:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c7015680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646754176:192.168.203.143@tcp:250:kworker.0 00000100:00000001:2.0:1787828682.599102:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828682.599104:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828682.599105:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787828682.599107:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1787828682.599110:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787828682.599112:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:2.0:1787828682.599114:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:2.0:1787828682.599117:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787828682.599119:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1787828682.599122:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787828682.599124:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828682.599127:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828682.599145:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1787828682.599147:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828682.599150:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787828682.599151:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787828682.599154:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:2.0:1787828682.599155:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00080000:3.0:1787828682.599159:0:40701:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3483 00000100:00000001:2.0:1787828682.599160:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787828682.599164:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787828682.599165:0:40701:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787828682.599169:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 02000000:00000001:2.0:1787828682.599171:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:3.0:1787828682.599171:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 02000000:00000001:2.0:1787828682.599175:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000020:00000001:3.0:1787828682.599175:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1787828682.599178:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 02000000:00000001:2.0:1787828682.599182:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828682.599183:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000020:00000001:3.0:1787828682.599188:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:3.0:1787828682.599190:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787828682.599193:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00100000:2.0:1787828682.599194:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c70143c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646754304:192.168.203.143@tcp:38:kworker.0 00000100:00080000:3.0:1787828682.599198:0:40701:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787828682.599201:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787828682.599204:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1787828682.599206:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828682.599209:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1787828682.599212:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828682.599220:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787828682.599222:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369536448 : -126234340015168 : ffff8d30c70143c0) 00000100:00000001:2.0:1787828682.599227:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:3.0:1787828682.599232:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787828682.599234:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787828682.599237:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787828682.599238:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828682.599240:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:3.0:1787828682.599242:0:40701:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1787828682.599245:0:40701:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:2.0:1787828682.599248:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:3.0:1787828682.599249:0:40701:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787828682.599256:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:2.0:1787828682.599257:0:2390:0:(events.c:34:request_out_callback()) Process entered 02000000:00000001:3.0:1787828682.599258:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828682.599261:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787828682.599261:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787828682.599262:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:2.0:1787828682.599264:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787828682.599266:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828682.599267:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1787828682.599271:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828682.599275:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828682.599276:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298155542 00000100:00000001:2.0:1787828682.599281:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298156800 00000100:00000001:2.0:1787828682.599283:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298155542 00000100:00100000:3.0:1787828682.599285:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c7014f00 x1874670646754432/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787828682.599286:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298156800 00000100:00000001:2.0:1787828682.599290:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787828682.599294:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828682.599298:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1787828682.599320:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828682/real 1787828682] req@ffff8d30c70143c0 x1874670646754304/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828707 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787828682.599321:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787828682.599326:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828682.599333:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:2.0:1787828682.599334:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00080000:3.0:1787828682.599336:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000001:2.0:1787828682.599337:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00080000:3.0:1787828682.599341:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1787828682.599342:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787828682.599343:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787828682.599357:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787828682.599358:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787828682.599359:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:2.0:1787828682.599360:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828682.599364:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00080000:3.0:1787828682.599364:0:40701:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3483 00000100:00000001:2.0:1787828682.599366:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:3.0:1787828682.599369:0:40701:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787828682.599373:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00100000:2.0:1787828682.599374:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c70143c0 x1874670646754304/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828707 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787828682.599376:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:3.0:1787828682.599381:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1787828682.599383:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1787828682.599385:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:2.0:1787828682.599390:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:3.0:1787828682.599390:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:3.0:1787828682.599392:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828682.599394:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787828682.599395:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:2.0:1787828682.599400:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:3.0:1787828682.599401:0:40701:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787828682.599405:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787828682.599406:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828682.599408:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:3.0:1787828682.599409:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:2.0:1787828682.599412:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 02000000:00000001:3.0:1787828682.599414:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787828682.599415:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 02000000:00000001:3.0:1787828682.599419:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787828682.599420:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00000001:3.0:1787828682.599422:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:3.0:1787828682.599425:0:40701:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00080000:2.0:1787828682.599426:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000008:00000001:3.0:1787828682.599428:0:40701:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828682.599430:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000001:3.0:1787828682.599431:0:40701:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787828682.599436:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787828682.599437:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828682.599438:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00100000:2.0:1787828682.599441:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c70143c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646754304:192.168.203.143@tcp:38:kworker.0 02000000:00000001:3.0:1787828682.599441:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787828682.599445:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828682.599447:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828682.599450:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828682.599451:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787828682.599453:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 00000100:00000001:3.0:1787828682.599454:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828682.599455:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787828682.599458:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:2.0:1787828682.599460:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:2.0:1787828682.599462:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787828682.599464:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00100000:3.0:1787828682.599464:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c7015a40 x1874670646754560/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787828682.599467:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787828682.599470:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828682.599474:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828682.599479:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1787828682.599481:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828682.599484:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787828682.599485:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787828682.599489:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787828682.599490:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828682.599491:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1787828682.599495:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787828682.599496:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828682.599497:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828682.599499:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787828682.599501:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828682.599503:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787828682.599507:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c7014f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646754432:192.168.203.143@tcp:38:kworker.0 00000100:00000001:2.0:1787828682.599509:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787828682.599511:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1787828682.599512:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828682.599514:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1787828682.599516:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828682.599519:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787828682.599521:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369539328 : -126234340012288 : ffff8d30c7014f00) 00000100:00000001:2.0:1787828682.599523:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1787828682.599532:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:2.0:1787828682.599536:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1787828682.599537:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828682.599538:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828682.599540:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1787828682.599541:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828682.599544:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298155542 00000100:00000001:2.0:1787828682.599546:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298156800 00000100:00000001:2.0:1787828682.599547:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298155542 00000100:00000001:2.0:1787828682.599548:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298156800 00000100:00000001:2.0:1787828682.599550:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787828682.599551:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828682.599552:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1787828682.599555:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828682/real 1787828682] req@ffff8d30c7014f00 x1874670646754432/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828707 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1787828682.599563:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787828682.599565:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1787828682.599567:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1787828682.599569:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787828682.599570:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828682.599572:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828682.599573:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787828682.599577:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c7014f00 x1874670646754432/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828707 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787828682.599585:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828682.599587:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787828682.599590:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828682.599591:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828682.599593:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828682.599595:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787828682.599596:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787828682.599628:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828682.599633:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:2.0:1787828682.599636:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1787828682.599640:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c7014f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646754432:192.168.203.143@tcp:38:kworker.0 00000100:00000001:2.0:1787828682.599645:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828682.599647:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828682.599649:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787828682.599651:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1787828682.599655:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787828682.599657:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:2.0:1787828682.599659:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:2.0:1787828682.599661:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787828682.599663:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1787828682.599666:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787828682.599668:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828682.599670:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828682.599673:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1787828682.599674:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828682.599675:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787828682.599676:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1787828682.599677:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1787828682.599678:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787828682.599680:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828682.599681:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787828682.599683:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828682.599684:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787828682.599686:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c7015a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646754560:192.168.203.143@tcp:8:kworker.0 00000100:00000001:2.0:1787828682.599689:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787828682.599690:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1787828682.599691:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828682.599692:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1787828682.599694:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828682.599699:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787828682.599700:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369542208 : -126234340009408 : ffff8d30c7015a40) 00000100:00000001:2.0:1787828682.599703:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1787828682.599712:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:2.0:1787828682.599715:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1787828682.599717:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828682.599718:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828682.599719:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1787828682.599721:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828682.599723:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298155556 00000100:00000001:2.0:1787828682.599725:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298156800 00000100:00000001:2.0:1787828682.599726:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298155556 00000100:00000001:2.0:1787828682.599728:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298156800 00000100:00000001:2.0:1787828682.599729:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787828682.599730:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828682.599732:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1787828682.599734:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828682/real 1787828682] req@ffff8d30c7015a40 x1874670646754560/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828707 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1787828682.599744:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787828682.599745:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1787828682.599746:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1787828682.599747:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787828682.599748:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828682.599749:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828682.599750:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787828682.599753:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c7015a40 x1874670646754560/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828707 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787828682.599757:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828682.599759:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787828682.599760:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828682.599761:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828682.599762:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828682.599763:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787828682.599764:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787828682.599766:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828682.599767:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:2.0:1787828682.599769:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1787828682.599770:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c7015a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646754560:192.168.203.143@tcp:8:kworker.0 00000100:00000001:2.0:1787828682.599772:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828682.599774:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828682.599774:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787828682.599775:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1787828682.599777:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787828682.599778:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:2.0:1787828682.599779:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:2.0:1787828682.599780:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787828682.599781:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1787828682.599782:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787828682.599783:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828682.599784:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828682.599786:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1787828682.599787:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828682.599788:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787828682.599789:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828683.494708:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828683.494715:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828683.494813:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828683.494819:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828683.494824:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828684.518710:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828684.518717:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828684.518770:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828684.518773:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828684.518776:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828685.542690:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828685.542696:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828685.542759:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828685.542764:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828685.542769:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828686.566675:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828686.566680:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828686.566727:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828686.566731:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828686.566733:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828687.590670:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828687.590675:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828687.590723:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828687.590727:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828687.590729:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787828687.718723:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828687.718731:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1787828687.718733:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787828687.718736:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787828687.718740:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787828687.718745:0:40701:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 3488 00000100:00080000:0.0:1787828687.718754:0:40701:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787828687.718756:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787828687.718757:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787828687.718760:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787828687.718762:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787828687.718765:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:0.0:1787828687.718766:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:0.0:1787828687.718768:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787828687.718769:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:0.0:1787828687.718772:0:40701:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787828687.718774:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828687.718776:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787828687.718778:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787828687.718781:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828687.718782:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1787828687.718784:0:40701:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828687.718794:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787828687.718796:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828687.718798:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787828687.718800:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787828687.718804:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828687.718821:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828687.718831:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d27c00 x1874670646754688/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787828687.718841:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787828687.718846:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828687.718861:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787828687.718865:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828687.718868:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1787828687.718870:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787828687.718872:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787828687.718875:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787828687.718878:0:40701:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3488 00000100:00080000:0.0:1787828687.718883:0:40701:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787828687.718885:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787828687.718886:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787828687.718888:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787828687.718889:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787828687.718891:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:2.0:1787828687.718892:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:0.0:1787828687.718893:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:0.0:1787828687.718897:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787828687.718898:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:2.0:1787828687.718899:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00080000:0.0:1787828687.718902:0:40701:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787828687.718905:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828687.718905:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 02000000:00000001:0.0:1787828687.718908:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:2.0:1787828687.718911:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 02000000:00000001:0.0:1787828687.718914:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787828687.718915:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828687.718918:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828687.718921:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 02000000:00000001:2.0:1787828687.718923:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000008:00100000:0.0:1787828687.718925:0:40701:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 02000000:00000001:2.0:1787828687.718928:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000008:00000001:0.0:1787828687.718929:0:40701:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828687.718933:0:40701:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828687.718936:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828687.718937:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787828687.718941:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828687.718942:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787828687.718944:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00100000:2.0:1787828687.718946:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d27c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646754688:192.168.203.143@tcp:250:kworker.0 02000000:00000001:0.0:1787828687.718947:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828687.718952:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828687.718955:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00100000:0.0:1787828687.718959:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d252c0 x1874670646754816/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:2.0:1787828687.718961:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1787828687.718965:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828687.718969:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:0.0:1787828687.718971:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787828687.718973:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828687.718974:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828687.718976:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787828687.718980:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828687.718982:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1787828687.718983:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787828687.718984:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787828687.718986:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787828687.718988:0:40701:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3488 00000100:00000001:2.0:1787828687.718989:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00080000:0.0:1787828687.718991:0:40701:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787828687.718992:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601351680 : -126234108199936 : ffff8d30d4d27c00) 00000100:00000001:0.0:1787828687.718994:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787828687.718996:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:2.0:1787828687.718997:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000020:00000001:0.0:1787828687.719001:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787828687.719003:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787828687.719004:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000020:00000001:0.0:1787828687.719006:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:0.0:1787828687.719007:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787828687.719008:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:0.0:1787828687.719012:0:40701:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787828687.719014:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828687.719016:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787828687.719019:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787828687.719022:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828687.719024:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1787828687.719026:0:40701:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1787828687.719029:0:40701:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:2.0:1787828687.719033:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:0.0:1787828687.719034:0:40701:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828687.719041:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787828687.719043:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828687.719045:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1787828687.719046:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787828687.719050:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:2.0:1787828687.719050:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 02000000:00000001:0.0:1787828687.719052:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828687.719054:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828687.719057:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787828687.719060:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828687.719063:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828687.719068:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d25680 x1874670646754944/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787828687.719071:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4298161920 00000100:00000001:0.0:1787828687.719079:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787828687.719081:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828687.719083:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828687.719085:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 00000100:00080000:0.0:1787828687.719086:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828687.719092:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1787828687.719093:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787828687.719094:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787828687.719096:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4298161920 00000100:00000001:0.0:1787828687.719097:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787828687.719101:0:40701:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3488 00000100:00000001:2.0:1787828687.719104:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 00000100:00080000:0.0:1787828687.719105:0:40701:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787828687.719109:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787828687.719111:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:2.0:1787828687.719112:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000020:00000001:0.0:1787828687.719113:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787828687.719115:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787828687.719118:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:2.0:1787828687.719118:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787828687.719123:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:0.0:1787828687.719125:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787828687.719126:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:0.0:1787828687.719128:0:40701:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787828687.719129:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828687.719131:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:2.0:1787828687.719132:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 02000000:00000001:0.0:1787828687.719133:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787828687.719136:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828687.719138:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1787828687.719141:0:40701:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787828687.719141:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828687/real 1787828687] req@ffff8d30d4d27c00 x1874670646754688/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828712 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00000001:0.0:1787828687.719146:0:40701:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828687.719150:0:40701:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828687.719156:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787828687.719158:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828687.719161:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000400:00000001:2.0:1787828687.719161:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 02000000:00000001:0.0:1787828687.719162:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787828687.719165:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:2.0:1787828687.719165:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787828687.719170:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828687.719171:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00100000:0.0:1787828687.719175:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d26940 x1874670646755072/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787828687.719175:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787828687.719177:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828687.719182:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828687.719185:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828687.719189:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787828687.719192:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787828687.719194:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d27c00 x1874670646754688/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828712 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787828687.719195:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828687.719204:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828687.719207:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787828687.719209:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828687.719210:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828687.719211:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828687.719213:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787828687.719215:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787828687.719217:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828687.719220:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:2.0:1787828687.719222:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1787828687.719225:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d27c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646754688:192.168.203.143@tcp:250:kworker.0 00000100:00000001:2.0:1787828687.719228:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828687.719230:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828687.719231:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787828687.719234:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1787828687.719236:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787828687.719238:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:2.0:1787828687.719239:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:2.0:1787828687.719242:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787828687.719243:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1787828687.719246:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787828687.719247:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828687.719249:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828687.719254:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1787828687.719256:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828687.719257:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787828687.719259:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1787828687.719260:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1787828687.719261:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787828687.719262:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828687.719263:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787828687.719265:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828687.719266:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787828687.719281:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d252c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646754816:192.168.203.143@tcp:38:kworker.0 00000100:00000001:2.0:1787828687.719284:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787828687.719285:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1787828687.719287:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828687.719288:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1787828687.719290:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828687.719295:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787828687.719296:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601341120 : -126234108210496 : ffff8d30d4d252c0) 00000100:00000001:2.0:1787828687.719299:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1787828687.719310:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:2.0:1787828687.719314:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1787828687.719316:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828687.719317:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828687.719319:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1787828687.719322:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828687.719326:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298160662 00000100:00000001:2.0:1787828687.719329:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298161920 00000100:00000001:2.0:1787828687.719331:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298160662 00000100:00000001:2.0:1787828687.719334:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298161920 00000100:00000001:2.0:1787828687.719336:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787828687.719338:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828687.719340:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1787828687.719345:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828687/real 1787828687] req@ffff8d30d4d252c0 x1874670646754816/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828712 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1787828687.719354:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787828687.719356:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1787828687.719359:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1787828687.719360:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787828687.719362:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828687.719364:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828687.719365:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787828687.719369:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d252c0 x1874670646754816/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828712 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787828687.719374:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828687.719376:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787828687.719377:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828687.719379:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828687.719379:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828687.719380:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787828687.719382:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787828687.719383:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828687.719385:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:2.0:1787828687.719387:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1787828687.719389:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d252c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646754816:192.168.203.143@tcp:38:kworker.0 00000100:00000001:2.0:1787828687.719391:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1787828687.719392:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787828687.719394:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828687.719395:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787828687.719397:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828687.719398:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787828687.719400:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d25680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646754944:192.168.203.143@tcp:38:kworker.0 00000100:00000001:2.0:1787828687.719402:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787828687.719404:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1787828687.719405:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828687.719406:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1787828687.719408:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828687.719412:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787828687.719413:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601342080 : -126234108209536 : ffff8d30d4d25680) 00000100:00000001:2.0:1787828687.719416:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1787828687.719424:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:2.0:1787828687.719428:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1787828687.719429:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828687.719430:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828687.719431:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1787828687.719433:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828687.719435:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298160662 00000100:00000001:2.0:1787828687.719437:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298161920 00000100:00000001:2.0:1787828687.719439:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298160662 00000100:00000001:2.0:1787828687.719440:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298161920 00000100:00000001:2.0:1787828687.719441:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787828687.719443:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828687.719444:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1787828687.719446:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828687/real 1787828687] req@ffff8d30d4d25680 x1874670646754944/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828712 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1787828687.719451:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787828687.719452:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1787828687.719453:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1787828687.719454:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787828687.719455:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828687.719456:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828687.719457:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787828687.719460:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d25680 x1874670646754944/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828712 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787828687.719465:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828687.719466:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787828687.719467:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828687.719468:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828687.719469:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828687.719470:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787828687.719471:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787828687.719472:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828687.719474:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:2.0:1787828687.719476:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1787828687.719477:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d25680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646754944:192.168.203.143@tcp:38:kworker.0 00000100:00000001:2.0:1787828687.719480:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1787828687.719480:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787828687.719481:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828687.719483:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787828687.719484:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828687.719485:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787828687.719487:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d26940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646755072:192.168.203.143@tcp:8:kworker.0 00000100:00000001:2.0:1787828687.719489:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787828687.719490:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1787828687.719491:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828687.719492:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1787828687.719494:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828687.719497:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787828687.719497:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601346880 : -126234108204736 : ffff8d30d4d26940) 00000100:00000001:2.0:1787828687.719500:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1787828687.719506:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:2.0:1787828687.719508:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1787828687.719509:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828687.719510:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828687.719512:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1787828687.719513:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828687.719515:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298160675 00000100:00000001:2.0:1787828687.719517:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298161920 00000100:00000001:2.0:1787828687.719518:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298160675 00000100:00000001:2.0:1787828687.719519:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298161920 00000100:00000001:2.0:1787828687.719521:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787828687.719522:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828687.719523:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1787828687.719525:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828687/real 1787828687] req@ffff8d30d4d26940 x1874670646755072/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828712 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1787828687.719530:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787828687.719531:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1787828687.719532:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1787828687.719533:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787828687.719534:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828687.719535:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828687.719535:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787828687.719538:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d26940 x1874670646755072/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828712 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787828687.719542:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828687.719543:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787828687.719545:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828687.719546:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828687.719546:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828687.719547:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787828687.719549:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787828687.719550:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828687.719553:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:2.0:1787828687.719554:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1787828687.719556:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d26940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646755072:192.168.203.143@tcp:8:kworker.0 00000100:00000001:2.0:1787828687.719559:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828687.719568:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828687.719569:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787828687.719570:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1787828687.719571:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787828687.719572:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:2.0:1787828687.719573:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:2.0:1787828687.719575:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787828687.719576:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1787828687.719577:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787828687.719578:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828687.719579:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828687.719580:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787828687.719581:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1787828687.719582:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787828687.719583:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:2.0:1787828687.719584:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:2.0:1787828687.719585:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787828687.719585:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1787828687.719586:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787828687.719587:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828687.719588:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828687.719589:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787828687.719590:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1787828687.719590:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787828687.719591:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:2.0:1787828687.719592:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:2.0:1787828687.719593:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787828687.719593:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1787828687.719595:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787828687.719596:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828687.719597:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828687.719625:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1787828687.719627:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828687.719629:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787828687.719631:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828688.614723:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828688.614731:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828688.615018:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828688.615024:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828688.615028:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828689.638724:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828689.638731:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828689.638781:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828689.638786:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828689.638790:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828690.662755:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828690.662765:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828690.662837:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828690.662842:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828690.662846:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828691.686689:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828691.686698:0:815:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1787828691.686700:0:815:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1787828691.686703:0:815:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828691.686708:0:815:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1787828691.686710:0:815:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828691.686712:0:815:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828691.686714:0:815:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828691.686805:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828691.686809:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828691.686812:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828692.710737:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828692.710745:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828692.710804:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828692.710809:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828692.710814:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787828692.838786:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828692.838802:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1787828692.838806:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1787828692.838810:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787828692.838816:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1787828692.838826:0:40701:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 3493 00000100:00080000:1.0:1787828692.838840:0:40701:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1787828692.838844:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787828692.838847:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:1.0:1787828692.838852:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:1.0:1787828692.838856:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:1.0:1787828692.838860:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:1.0:1787828692.838863:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:1.0:1787828692.838866:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787828692.838868:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:1.0:1787828692.838874:0:40701:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:1.0:1787828692.838878:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828692.838881:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1787828692.838886:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787828692.838891:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828692.838894:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:1.0:1787828692.838898:0:40701:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828692.838914:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1787828692.838917:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828692.838921:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1787828692.838925:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1787828692.838931:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828692.838946:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787828692.838959:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d13eed00 x1874670646755200/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787828692.838974:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787828692.838980:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828692.839000:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1787828692.839004:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828692.839009:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1787828692.839011:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1787828692.839013:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787828692.839017:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1787828692.839020:0:40701:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3493 00000100:00080000:1.0:1787828692.839026:0:40701:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1787828692.839029:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787828692.839031:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:1.0:1787828692.839035:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:1.0:1787828692.839037:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:1.0:1787828692.839040:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000020:00000001:1.0:1787828692.839044:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:1.0:1787828692.839047:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787828692.839049:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:1.0:1787828692.839054:0:40701:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:1.0:1787828692.839058:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828692.839059:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:1.0:1787828692.839061:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:2.0:1787828692.839065:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1787828692.839069:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 02000000:00000001:1.0:1787828692.839071:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787828692.839073:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787828692.839077:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828692.839079:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828692.839083:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:1.0:1787828692.839084:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 02000000:00000001:2.0:1787828692.839088:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828692.839092:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000008:00100000:1.0:1787828692.839094:0:40701:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828692.839098:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d13eed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646755200:192.168.203.143@tcp:250:kworker.0 00000008:00000001:1.0:1787828692.839101:0:40701:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828692.839105:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:1.0:1787828692.839107:0:40701:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828692.839112:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787828692.839116:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787828692.839117:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828692.839121:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828692.839122:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:1.0:1787828692.839128:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787828692.839130:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828692.839131:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1787828692.839137:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828692.839146:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828692.839153:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00100000:1.0:1787828692.839156:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d13ef480 x1874670646755328/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787828692.839157:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839541349632 : -126234168201984 : ffff8d30d13eed00) 00000100:00000001:2.0:1787828692.839165:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:1.0:1787828692.839175:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787828692.839177:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828692.839181:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1787828692.839183:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828692.839187:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:1.0:1787828692.839188:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1787828692.839190:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787828692.839192:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1787828692.839194:0:40701:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3493 00000100:00080000:1.0:1787828692.839199:0:40701:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1787828692.839200:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787828692.839202:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:1.0:1787828692.839205:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:1.0:1787828692.839207:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:1.0:1787828692.839209:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000020:00000001:1.0:1787828692.839210:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:1.0:1787828692.839212:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787828692.839213:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:1.0:1787828692.839217:0:40701:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:1.0:1787828692.839219:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828692.839221:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1787828692.839224:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787828692.839225:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828692.839227:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:1.0:1787828692.839229:0:40701:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1787828692.839231:0:40701:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828692.839248:0:40701:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828692.839254:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1787828692.839255:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828692.839257:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1787828692.839258:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:2.0:1787828692.839260:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828692.839261:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828692.839289:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4298167040 00000100:00000001:1.0:1787828692.839290:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787828692.839301:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d13ef840 x1874670646755456/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787828692.839310:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 00000100:00000001:1.0:1787828692.839313:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787828692.839318:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828692.839323:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828692.839324:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4298167040 00000100:00080000:1.0:1787828692.839327:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828692.839336:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1787828692.839339:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:2.0:1787828692.839341:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 00000100:00080000:1.0:1787828692.839344:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787828692.839346:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787828692.839352:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:2.0:1787828692.839353:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828692.839360:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1787828692.839363:0:40701:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3493 00000100:00000001:2.0:1787828692.839364:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:1.0:1787828692.839373:0:40701:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787828692.839377:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787828692.839378:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828692.839381:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:1.0:1787828692.839383:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:2.0:1787828692.839384:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787828692.839387:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000020:00000001:1.0:1787828692.839392:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:2.0:1787828692.839392:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1787828692.839396:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000020:00000001:1.0:1787828692.839398:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:2.0:1787828692.839399:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787828692.839405:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 02000000:00000001:2.0:1787828692.839405:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787828692.839408:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000020:00000001:1.0:1787828692.839413:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 02000000:00000001:2.0:1787828692.839413:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828692.839422:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00100000:2.0:1787828692.839424:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d13ef480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646755328:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787828692.839426:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:2.0:1787828692.839431:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787828692.839436:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000100:00080000:1.0:1787828692.839440:0:40701:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 02000000:00000001:2.0:1787828692.839440:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828692.839445:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:1.0:1787828692.839448:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828692.839450:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828692.839456:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1787828692.839465:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787828692.839465:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787828692.839468:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839541351552 : -126234168200064 : ffff8d30d13ef480) 02000000:00000001:1.0:1787828692.839472:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828692.839475:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:1.0:1787828692.839478:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:1.0:1787828692.839484:0:40701:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000008:00000001:1.0:1787828692.839504:0:40701:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828692.839507:0:40701:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828692.839509:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828692.839514:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:2.0:1787828692.839516:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298165782 02000000:00000001:1.0:1787828692.839518:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828692.839521:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298167040 00000100:00000001:1.0:1787828692.839524:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:2.0:1787828692.839525:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298165782 02000000:00000001:1.0:1787828692.839529:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:2.0:1787828692.839530:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298167040 02000000:00000001:1.0:1787828692.839535:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828692.839536:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787828692.839543:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828692.839546:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1787828692.839548:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787828692.839549:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828692.839550:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828692.839558:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000100:00100000:1.0:1787828692.839561:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d13ede00 x1874670646755584/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:2.0:1787828692.839563:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828692.839587:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787828692.839599:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d13ef840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646755456:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787828692.839601:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787828692.839608:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828692.839610:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787828692.839615:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:1.0:1787828692.839616:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828692.839619:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828692.839624:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1787828692.839629:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828692.839636:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787828692.839638:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839541352512 : -126234168199104 : ffff8d30d13ef840) 00000100:00000001:2.0:1787828692.839642:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:2.0:1787828692.839693:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828692.839698:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298165783 00000100:00000001:2.0:1787828692.839700:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298167040 00000100:00000001:2.0:1787828692.839703:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298165783 00000100:00000001:2.0:1787828692.839706:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298167040 00000100:00000001:2.0:1787828692.839709:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787828692.839727:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828692.839730:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828692.839733:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828692.839737:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1787828692.839739:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:2.0:1787828692.839741:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787828692.839743:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1787828692.839745:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1787828692.839747:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787828692.839748:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828692.839759:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787828692.839761:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828692.839764:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787828692.839768:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d13ede00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646755584:192.168.203.143@tcp:8:kworker.0 00000100:00000001:2.0:1787828692.839772:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787828692.839776:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1787828692.839777:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828692.839780:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1787828692.839783:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828692.839791:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787828692.839793:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839541345792 : -126234168205824 : ffff8d30d13ede00) 00000100:00000001:2.0:1787828692.839798:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:2.0:1787828692.839813:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828692.839817:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298165796 00000100:00000001:2.0:1787828692.839821:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298167040 00000100:00000001:2.0:1787828692.839824:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298165796 00000100:00000001:2.0:1787828692.839826:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298167040 00000100:00000001:2.0:1787828692.839829:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787828692.839832:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828692.839835:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828692.839837:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828692.839842:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1787828692.839843:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:2.0:1787828692.839846:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787828692.839847:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1787828692.839849:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828692.839850:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828693.734704:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828693.734710:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828693.734779:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828693.734783:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828693.734787:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828694.758677:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828694.758682:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828694.758781:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828694.758785:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828694.758789:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828695.782738:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828695.782745:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828695.782835:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828695.782839:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828695.782852:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:3.0:1787828695.910942:0:2381:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.143:988 00000400:00000100:3.0:1787828695.910996:0:2381:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.143@tcp at host 192.168.203.143 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:3.0:1787828695.911017:0:2381:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.143@tcp 00000100:00000001:3.0:1787828695.911036:0:2381:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787828695.911062:0:2381:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787828695.911067:0:2381:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828695.911070:0:2381:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:3.0:1787828695.911083:0:2381:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.143@tcp 00000100:00000001:3.0:1787828695.911091:0:2381:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787828695.911095:0:2381:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787828695.911098:0:2381:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828695.911101:0:2381:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:3.0:1787828695.911110:0:2381:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.143@tcp 00000100:00000001:3.0:1787828695.911119:0:2381:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1787828695.911124:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1787828695.911125:0:2381:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787828695.911131:0:2381:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828695.911133:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787828695.911135:0:2381:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1787828695.911139:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000800:00000100:3.0:1787828695.911148:0:2381:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.143@tcp 00000100:00000001:3.0:1787828695.911157:0:2381:0:(events.c:34:request_out_callback()) Process entered 00000100:00100000:2.0:1787828695.911160:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828692/real 1787828695] req@ffff8d30d13ef480 x1874670646755328/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828717 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787828695.911162:0:2381:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787828695.911167:0:2381:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828695.911172:0:2381:0:(events.c:68:request_out_callback()) Process leaving 00000400:00000001:2.0:1787828695.911188:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787828695.911193:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1787828695.911201:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1787828695.911204:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787828695.911207:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828695.911231:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828695.911234:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787828695.911243:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d13ef480 x1874670646755328/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828717 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787828695.911254:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828695.911258:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787828695.911263:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828695.911265:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828695.911267:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828695.911271:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787828695.911274:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787828695.911279:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828695.911286:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:2.0:1787828695.911290:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1787828695.911295:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d13ef480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646755328:192.168.203.143@tcp:38:kworker.0 00000100:00000001:2.0:1787828695.911302:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1787828695.911308:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828692/real 1787828695] req@ffff8d30d13ef840 x1874670646755456/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828717 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1787828695.911321:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787828695.911323:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1787828695.911328:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1787828695.911330:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787828695.911332:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828695.911335:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828695.911337:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787828695.911342:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d13ef840 x1874670646755456/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828717 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787828695.911377:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828695.911380:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787828695.911383:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828695.911385:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828695.911388:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828695.911390:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787828695.911392:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787828695.911396:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828695.911401:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:2.0:1787828695.911404:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1787828695.911409:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d13ef840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646755456:192.168.203.143@tcp:38:kworker.0 00000100:00000001:2.0:1787828695.911414:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1787828695.911419:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828692/real 1787828695] req@ffff8d30d13eed00 x1874670646755200/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828717 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1787828695.911430:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787828695.911432:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1787828695.911437:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1787828695.911440:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787828695.911442:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828695.911444:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828695.911446:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787828695.911452:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d13eed00 x1874670646755200/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828717 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787828695.911461:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828695.911464:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787828695.911467:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828695.911470:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828695.911471:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828695.911475:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787828695.911477:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787828695.911481:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828695.911486:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:2.0:1787828695.911489:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1787828695.911494:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d13eed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646755200:192.168.203.143@tcp:250:kworker.0 00000100:00000001:2.0:1787828695.911500:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828695.911503:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828695.911506:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787828695.911509:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1787828695.911516:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787828695.911520:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:2.0:1787828695.911523:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:2.0:1787828695.911527:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787828695.911530:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1787828695.911534:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787828695.911542:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828695.911545:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828695.911547:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787828695.911549:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1787828695.911552:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787828695.911554:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:2.0:1787828695.911556:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:2.0:1787828695.911559:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787828695.911561:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1787828695.911563:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787828695.911565:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828695.911568:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828695.911569:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787828695.911571:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1787828695.911573:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787828695.911575:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:2.0:1787828695.911576:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:2.0:1787828695.911579:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787828695.911581:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1787828695.911584:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787828695.911585:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828695.911588:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828695.911591:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787828695.911593:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1787828695.911595:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1787828695.911641:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828692/real 1787828695] req@ffff8d30d13ede00 x1874670646755584/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828717 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1787828695.911653:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787828695.911655:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1787828695.911661:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1787828695.911664:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787828695.911666:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828695.911669:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828695.911671:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787828695.911677:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d13ede00 x1874670646755584/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828717 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787828695.911684:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828695.911687:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787828695.911690:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828695.911691:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828695.911693:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828695.911695:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787828695.911697:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787828695.911699:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828695.911704:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:2.0:1787828695.911706:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1787828695.911709:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d13ede00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646755584:192.168.203.143@tcp:8:kworker.0 00000100:00000001:2.0:1787828695.911735:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828695.911737:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828695.911738:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787828695.911740:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1787828695.911743:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787828695.911744:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:2.0:1787828695.911746:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:2.0:1787828695.911749:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787828695.911751:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1787828695.911754:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787828695.911755:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828695.911758:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828695.911766:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1787828695.911768:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828695.911771:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787828695.911772:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828696.806797:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828696.806805:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828696.806918:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828696.806927:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828696.806934:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828697.830751:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828697.830756:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828697.830895:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828697.830903:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828697.830908:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787828697.958740:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828697.958751:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1787828697.958755:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1787828697.958759:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787828697.958765:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1787828697.958773:0:40701:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 3498 00000100:00080000:1.0:1787828697.958786:0:40701:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1787828697.958790:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787828697.958792:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:1.0:1787828697.958797:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:1.0:1787828697.958800:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:1.0:1787828697.958804:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:1.0:1787828697.958806:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:1.0:1787828697.958808:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787828697.958809:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:1.0:1787828697.958814:0:40701:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:1.0:1787828697.958817:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828697.958819:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1787828697.958823:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787828697.958826:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828697.958828:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:1.0:1787828697.958830:0:40701:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828697.958843:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1787828697.958845:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828697.958850:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1787828697.958853:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1787828697.958860:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828697.958877:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787828697.958893:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d13ef0c0 x1874670646755712/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787828697.958907:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787828697.958914:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828697.958940:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1787828697.958943:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828697.958949:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1787828697.958950:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1787828697.958952:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787828697.958954:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1787828697.958957:0:40701:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3498 00000100:00080000:1.0:1787828697.958961:0:40701:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1787828697.958969:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787828697.958971:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:1.0:1787828697.958973:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:1.0:1787828697.958974:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:1.0:1787828697.958977:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000020:00000001:1.0:1787828697.958979:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:1.0:1787828697.958980:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787828697.958981:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:1.0:1787828697.958984:0:40701:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:1.0:1787828697.958988:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828697.958990:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1787828697.958997:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787828697.958999:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828697.959001:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:1.0:1787828697.959007:0:40701:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1787828697.959010:0:40701:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828697.959012:0:40701:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828697.959018:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1787828697.959020:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828697.959024:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1787828697.959027:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1787828697.959034:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828697.959043:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787828697.959054:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d13efc00 x1874670646755840/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787828697.959056:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787828697.959085:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1787828697.959093:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787828697.959094:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787828697.959101:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828697.959101:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787828697.959108:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828697.959112:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828697.959117:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000100:00080000:1.0:1787828697.959120:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 02000000:00000001:2.0:1787828697.959125:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00080000:1.0:1787828697.959128:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 02000000:00000001:2.0:1787828697.959133:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828697.959140:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1787828697.959146:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1787828697.959148:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d13ef0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646755712:192.168.203.143@tcp:250:kworker.0 00000100:00000001:1.0:1787828697.959154:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1787828697.959164:0:40701:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3498 00000100:00000001:2.0:1787828697.959166:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00080000:1.0:1787828697.959174:0:40701:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1787828697.959179:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787828697.959181:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:1.0:1787828697.959185:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:1.0:1787828697.959187:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:1.0:1787828697.959189:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000020:00000001:1.0:1787828697.959192:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:1.0:1787828697.959194:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787828697.959196:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:1.0:1787828697.959219:0:40701:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:1.0:1787828697.959222:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828697.959225:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1787828697.959229:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787828697.959232:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828697.959235:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:1.0:1787828697.959238:0:40701:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1787828697.959241:0:40701:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828697.959243:0:40701:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828697.959249:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1787828697.959252:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828697.959255:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1787828697.959257:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1787828697.959259:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828697.959269:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787828697.959275:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d13ed2c0 x1874670646755968/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787828697.959285:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787828697.959294:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828697.959298:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1787828697.959301:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828697.959306:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1787828697.959308:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1787828697.959310:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787828697.959312:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1787828697.959316:0:40701:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3498 00000100:00080000:1.0:1787828697.959321:0:40701:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1787828697.959324:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787828697.959326:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:1.0:1787828697.959329:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:1.0:1787828697.959331:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:1.0:1787828697.959333:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000020:00000001:1.0:1787828697.959336:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:1.0:1787828697.959339:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787828697.959340:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:1.0:1787828697.959345:0:40701:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:1.0:1787828697.959347:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828697.959349:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1787828697.959353:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787828697.959355:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828697.959358:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:1.0:1787828697.959361:0:40701:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000008:00000001:1.0:1787828697.959364:0:40701:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828697.959366:0:40701:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828697.959371:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1787828697.959373:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828697.959375:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1787828697.959377:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1787828697.959379:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828697.959385:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787828697.959392:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d13ed680 x1874670646756096/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787828697.959401:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787828697.959403:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828697.959406:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828697.959417:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1787828697.959423:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828697.959427:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1787828697.959439:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828697.959458:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787828697.959460:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839541350592 : -126234168201024 : ffff8d30d13ef0c0) 00000100:00000001:2.0:1787828697.959465:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1787828697.959690:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:2.0:1787828697.959704:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1787828697.959710:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828697.959713:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828697.959714:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1787828697.959720:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828697.959726:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4298172161 00000100:00000001:2.0:1787828697.959737:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 00000100:00000001:2.0:1787828697.959761:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4298172161 00000100:00000001:2.0:1787828697.959766:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 00000100:00000001:2.0:1787828697.959770:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787828697.959773:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828697.959776:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1787828697.959784:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828697/real 1787828697] req@ffff8d30d13ef0c0 x1874670646755712/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828722 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1787828697.959796:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787828697.959798:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1787828697.959803:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1787828697.959806:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787828697.959808:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828697.959811:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828697.959813:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787828697.959818:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d13ef0c0 x1874670646755712/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828722 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787828697.959827:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828697.959830:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787828697.959833:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828697.959835:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828697.959837:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828697.959840:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787828697.959842:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787828697.959846:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828697.959853:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:2.0:1787828697.959858:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1787828697.959864:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d13ef0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646755712:192.168.203.143@tcp:250:kworker.0 00000100:00000001:2.0:1787828697.959869:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828697.959873:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828697.959876:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787828697.959881:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1787828697.959888:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787828697.959892:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:2.0:1787828697.959894:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:2.0:1787828697.959899:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787828697.959901:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1787828697.959904:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787828697.959906:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828697.959908:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828697.959916:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1787828697.959918:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828697.959921:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787828697.959923:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1787828697.959924:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1787828697.959926:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787828697.959929:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828697.959982:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787828697.959984:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828697.959986:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787828697.959990:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d13efc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646755840:192.168.203.143@tcp:38:kworker.0 00000100:00000001:2.0:1787828697.959993:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787828697.959995:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1787828697.959996:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828697.959998:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1787828697.960000:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828697.960006:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787828697.960007:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839541353472 : -126234168198144 : ffff8d30d13efc00) 00000100:00000001:2.0:1787828697.960011:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1787828697.960024:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:2.0:1787828697.960032:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1787828697.960035:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828697.960036:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828697.960038:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1787828697.960041:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828697.960045:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298170903 00000100:00000001:2.0:1787828697.960049:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298172161 00000100:00000001:2.0:1787828697.960051:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298170903 00000100:00000001:2.0:1787828697.960054:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298172161 00000100:00000001:2.0:1787828697.960057:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787828697.960059:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828697.960061:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1787828697.960065:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828697/real 1787828697] req@ffff8d30d13efc00 x1874670646755840/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828722 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1787828697.960075:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787828697.960077:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1787828697.960081:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1787828697.960083:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787828697.960085:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828697.960087:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828697.960088:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787828697.960092:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d13efc00 x1874670646755840/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828722 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787828697.960101:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828697.960104:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787828697.960107:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828697.960109:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828697.960110:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828697.960112:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787828697.960115:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787828697.960119:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828697.960123:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:2.0:1787828697.960127:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1787828697.960132:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d13efc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646755840:192.168.203.143@tcp:38:kworker.0 00000100:00000001:2.0:1787828697.960139:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1787828697.960141:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787828697.960143:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828697.960147:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787828697.960148:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828697.960150:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787828697.960153:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d13ed2c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646755968:192.168.203.143@tcp:38:kworker.0 00000100:00000001:2.0:1787828697.960155:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787828697.960157:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1787828697.960159:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828697.960160:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1787828697.960162:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828697.960167:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787828697.960169:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839541342912 : -126234168208704 : ffff8d30d13ed2c0) 00000100:00000001:2.0:1787828697.960174:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1787828697.960191:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:2.0:1787828697.960197:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1787828697.960211:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828697.960213:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828697.960214:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1787828697.960218:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828697.960222:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298170903 00000100:00000001:2.0:1787828697.960225:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298172161 00000100:00000001:2.0:1787828697.960227:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298170903 00000100:00000001:2.0:1787828697.960230:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298172161 00000100:00000001:2.0:1787828697.960232:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787828697.960234:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828697.960236:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1787828697.960241:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828697/real 1787828697] req@ffff8d30d13ed2c0 x1874670646755968/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828722 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1787828697.960249:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787828697.960251:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1787828697.960254:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1787828697.960256:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787828697.960257:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828697.960259:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828697.960261:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787828697.960265:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d13ed2c0 x1874670646755968/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828722 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787828697.960278:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828697.960281:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787828697.960283:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828697.960285:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828697.960286:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828697.960288:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787828697.960290:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787828697.960292:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828697.960296:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:2.0:1787828697.960298:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1787828697.960302:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d13ed2c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646755968:192.168.203.143@tcp:38:kworker.0 00000100:00000001:2.0:1787828697.960306:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1787828697.960308:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787828697.960309:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828697.960311:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787828697.960313:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828697.960315:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787828697.960319:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d13ed680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646756096:192.168.203.143@tcp:8:kworker.0 00000100:00000001:2.0:1787828697.960328:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787828697.960330:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1787828697.960332:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828697.960334:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1787828697.960337:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828697.960345:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787828697.960346:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839541343872 : -126234168207744 : ffff8d30d13ed680) 00000100:00000001:2.0:1787828697.960350:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1787828697.960363:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:2.0:1787828697.960368:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1787828697.960370:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828697.960371:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828697.960374:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1787828697.960377:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828697.960380:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298170916 00000100:00000001:2.0:1787828697.960383:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298172161 00000100:00000001:2.0:1787828697.960385:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298170916 00000100:00000001:2.0:1787828697.960386:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298172161 00000100:00000001:2.0:1787828697.960388:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787828697.960389:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828697.960391:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1787828697.960394:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828697/real 1787828697] req@ffff8d30d13ed680 x1874670646756096/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828722 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1787828697.960403:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787828697.960404:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1787828697.960407:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1787828697.960408:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787828697.960410:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828697.960411:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828697.960413:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787828697.960417:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d13ed680 x1874670646756096/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828722 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787828697.960424:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828697.960426:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787828697.960429:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828697.960430:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828697.960432:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828697.960434:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787828697.960435:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787828697.960438:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828697.960441:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:2.0:1787828697.960443:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1787828697.960447:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d13ed680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646756096:192.168.203.143@tcp:8:kworker.0 00000100:00000001:2.0:1787828697.960450:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828697.960457:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828697.960459:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787828697.960461:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1787828697.960463:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787828697.960465:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:2.0:1787828697.960467:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:2.0:1787828697.960470:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787828697.960471:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1787828697.960473:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787828697.960474:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828697.960476:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828697.960477:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787828697.960478:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1787828697.960480:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787828697.960481:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:2.0:1787828697.960482:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:2.0:1787828697.960485:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787828697.960486:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1787828697.960489:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787828697.960490:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828697.960492:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828697.960493:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787828697.960494:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1787828697.960496:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787828697.960497:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:2.0:1787828697.960498:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:2.0:1787828697.960500:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787828697.960501:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1787828697.960502:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787828697.960504:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828697.960505:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828697.960510:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1787828697.960511:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828697.960513:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787828697.960515:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828698.854710:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828698.854718:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828698.854841:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828698.854844:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828698.854847:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828699.878712:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828699.878717:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828699.878767:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828699.878770:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828699.878773:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828700.902754:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828700.902761:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828700.902832:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828700.902838:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828700.902843:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828701.926763:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828701.926787:0:815:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1787828701.926790:0:815:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1787828701.926793:0:815:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828701.926797:0:815:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1787828701.926800:0:815:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828701.926802:0:815:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828701.926804:0:815:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828701.926928:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828701.926933:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828701.926963:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828702.950698:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828702.950703:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828702.950811:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828702.950816:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828702.950820:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787828703.078711:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828703.078720:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1787828703.078722:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1787828703.078725:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787828703.078729:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1787828703.078735:0:40701:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 3503 00000100:00080000:1.0:1787828703.078745:0:40701:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1787828703.078747:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787828703.078749:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:1.0:1787828703.078752:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:1.0:1787828703.078754:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:1.0:1787828703.078757:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:1.0:1787828703.078759:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:1.0:1787828703.078760:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787828703.078761:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:1.0:1787828703.078765:0:40701:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:1.0:1787828703.078767:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828703.078769:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1787828703.078773:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787828703.078775:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828703.078777:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:1.0:1787828703.078779:0:40701:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828703.078789:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1787828703.078791:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828703.078793:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1787828703.078795:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1787828703.078799:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828703.078808:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787828703.078815:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d13ec3c0 x1874670646756224/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787828703.078823:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787828703.078827:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828703.078840:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1787828703.078842:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828703.078845:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1787828703.078846:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1787828703.078847:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787828703.078849:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1787828703.078851:0:40701:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3503 00000100:00080000:1.0:1787828703.078853:0:40701:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1787828703.078854:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787828703.078856:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:1.0:1787828703.078857:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:1.0:1787828703.078858:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:1.0:1787828703.078860:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000020:00000001:1.0:1787828703.078862:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:1.0:1787828703.078864:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787828703.078865:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:1.0:1787828703.078867:0:40701:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:1.0:1787828703.078869:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828703.078871:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:2.0:1787828703.078874:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:1.0:1787828703.078876:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787828703.078879:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 02000000:00000001:1.0:1787828703.078880:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828703.078884:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:2.0:1787828703.078884:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1787828703.078888:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000008:00100000:1.0:1787828703.078891:0:40701:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00000001:2.0:1787828703.078892:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:1.0:1787828703.078896:0:40701:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828703.078899:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787828703.078902:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828703.078904:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828703.078907:0:40701:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787828703.078911:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d13ec3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646756224:192.168.203.143@tcp:250:kworker.0 02000000:00000001:1.0:1787828703.078912:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1787828703.078915:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828703.078918:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:2.0:1787828703.078918:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787828703.078921:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1787828703.078923:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828703.078923:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:1.0:1787828703.078927:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828703.078927:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828703.078929:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00100000:1.0:1787828703.078934:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d13ecb40 x1874670646756352/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:2.0:1787828703.078935:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828703.078943:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787828703.078946:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828703.078948:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1787828703.078950:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:2.0:1787828703.078952:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00080000:1.0:1787828703.078954:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1787828703.078955:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839541339072 : -126234168212544 : ffff8d30d13ec3c0) 00000100:00000001:1.0:1787828703.078957:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1787828703.078960:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787828703.078960:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:1.0:1787828703.078964:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1787828703.078966:0:40701:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3503 00000100:00080000:1.0:1787828703.078968:0:40701:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1787828703.078969:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787828703.078970:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:1.0:1787828703.078972:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:1.0:1787828703.078973:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:1.0:1787828703.078974:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000020:00000001:1.0:1787828703.078976:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:1.0:1787828703.078978:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787828703.078979:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:1.0:1787828703.078982:0:40701:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:1.0:1787828703.078984:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828703.078986:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1787828703.078988:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787828703.078989:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828703.078990:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:1.0:1787828703.078992:0:40701:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1787828703.078996:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000008:00000001:1.0:1787828703.078997:0:40701:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828703.079002:0:40701:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828703.079005:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1787828703.079007:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828703.079008:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1787828703.079009:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1787828703.079011:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828703.079011:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1787828703.079017:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787828703.079018:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828703.079021:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828703.079025:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00100000:1.0:1787828703.079026:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d13ec000 x1874670646756480/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787828703.079030:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828703.079036:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787828703.079036:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4298177280 00000100:00000001:1.0:1787828703.079055:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828703.079058:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1787828703.079059:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828703.079061:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1787828703.079062:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1787828703.079064:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787828703.079064:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 00000100:00000001:1.0:1787828703.079069:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1787828703.079071:0:40701:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3503 00000100:00080000:1.0:1787828703.079074:0:40701:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1787828703.079075:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787828703.079076:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:1.0:1787828703.079077:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:1.0:1787828703.079079:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:2.0:1787828703.079079:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4298177280 00000020:00000001:1.0:1787828703.079081:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000020:00000001:1.0:1787828703.079085:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:2.0:1787828703.079086:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 00000100:00000001:1.0:1787828703.079087:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787828703.079091:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:2.0:1787828703.079092:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00080000:1.0:1787828703.079095:0:40701:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787828703.079097:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828703.079100:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828703.079101:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 02000000:00000001:1.0:1787828703.079103:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1787828703.079107:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787828703.079110:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787828703.079110:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828703/real 1787828703] req@ffff8d30d13ec3c0 x1874670646756224/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828728 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787828703.079114:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:1.0:1787828703.079134:0:40701:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000008:00000001:1.0:1787828703.079138:0:40701:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828703.079142:0:40701:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:2.0:1787828703.079144:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 02000000:00000001:1.0:1787828703.079146:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000400:00000001:2.0:1787828703.079147:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 02000000:00000001:1.0:1787828703.079149:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828703.079152:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:2.0:1787828703.079153:0:2390:0:(events.c:78:reply_in_callback()) Process entered 02000000:00000001:1.0:1787828703.079154:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:2.0:1787828703.079156:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 02000000:00000001:1.0:1787828703.079157:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828703.079160:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828703.079175:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787828703.079176:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828703.079177:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787828703.079184:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d13ec780 x1874670646756608/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787828703.079185:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d13ec3c0 x1874670646756224/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828728 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787828703.079200:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787828703.079202:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828703.079204:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828703.079211:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828703.079216:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787828703.079218:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828703.079220:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828703.079221:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828703.079223:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787828703.079224:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787828703.079226:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828703.079230:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:2.0:1787828703.079232:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1787828703.079235:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d13ec3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646756224:192.168.203.143@tcp:250:kworker.0 00000100:00000001:2.0:1787828703.079240:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828703.079242:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828703.079243:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787828703.079245:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1787828703.079248:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787828703.079249:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:2.0:1787828703.079251:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:2.0:1787828703.079255:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787828703.079256:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1787828703.079258:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787828703.079259:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828703.079261:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828703.079266:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1787828703.079268:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828703.079269:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787828703.079270:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1787828703.079272:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1787828703.079273:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787828703.079274:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828703.079275:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787828703.079277:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828703.079278:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787828703.079280:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d13ecb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646756352:192.168.203.143@tcp:38:kworker.0 00000100:00000001:2.0:1787828703.079283:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787828703.079284:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1787828703.079285:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828703.079287:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1787828703.079289:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828703.079299:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787828703.079300:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839541340992 : -126234168210624 : ffff8d30d13ecb40) 00000100:00000001:2.0:1787828703.079303:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1787828703.079313:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:2.0:1787828703.079316:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1787828703.079318:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828703.079319:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828703.079320:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1787828703.079322:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828703.079341:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298176022 00000100:00000001:2.0:1787828703.079344:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298177280 00000100:00000001:2.0:1787828703.079345:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298176022 00000100:00000001:2.0:1787828703.079347:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298177280 00000100:00000001:2.0:1787828703.079349:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787828703.079350:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828703.079351:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1787828703.079354:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828703/real 1787828703] req@ffff8d30d13ecb40 x1874670646756352/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828728 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1787828703.079360:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787828703.079361:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1787828703.079362:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1787828703.079363:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787828703.079364:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828703.079366:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828703.079366:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787828703.079369:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d13ecb40 x1874670646756352/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828728 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787828703.079374:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828703.079375:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787828703.079376:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828703.079382:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828703.079383:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828703.079384:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787828703.079385:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787828703.079387:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828703.079389:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:2.0:1787828703.079390:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1787828703.079392:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d13ecb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646756352:192.168.203.143@tcp:38:kworker.0 00000100:00000001:2.0:1787828703.079395:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1787828703.079396:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787828703.079397:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828703.079398:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787828703.079399:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828703.079401:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787828703.079403:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d13ec000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646756480:192.168.203.143@tcp:38:kworker.0 00000100:00000001:2.0:1787828703.079405:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787828703.079406:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1787828703.079407:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828703.079408:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1787828703.079410:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828703.079413:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787828703.079414:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839541338112 : -126234168213504 : ffff8d30d13ec000) 00000100:00000001:2.0:1787828703.079428:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1787828703.079435:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:2.0:1787828703.079438:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1787828703.079439:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828703.079440:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828703.079441:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1787828703.079443:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828703.079444:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298176022 00000100:00000001:2.0:1787828703.079446:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298177280 00000100:00000001:2.0:1787828703.079448:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298176022 00000100:00000001:2.0:1787828703.079449:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298177280 00000100:00000001:2.0:1787828703.079450:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787828703.079452:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828703.079453:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1787828703.079455:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828703/real 1787828703] req@ffff8d30d13ec000 x1874670646756480/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828728 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1787828703.079460:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787828703.079460:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1787828703.079462:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1787828703.079462:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787828703.079463:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828703.079464:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828703.079465:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787828703.079468:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d13ec000 x1874670646756480/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828728 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787828703.079472:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828703.079473:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787828703.079475:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828703.079476:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828703.079476:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828703.079477:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787828703.079478:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787828703.079480:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828703.079481:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:2.0:1787828703.079483:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1787828703.079484:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d13ec000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646756480:192.168.203.143@tcp:38:kworker.0 00000100:00000001:2.0:1787828703.079486:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1787828703.079487:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787828703.079488:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828703.079489:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787828703.079490:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828703.079491:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787828703.079493:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d13ec780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646756608:192.168.203.143@tcp:8:kworker.0 00000100:00000001:2.0:1787828703.079495:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787828703.079496:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1787828703.079496:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828703.079497:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1787828703.079499:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828703.079502:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787828703.079503:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839541340032 : -126234168211584 : ffff8d30d13ec780) 00000100:00000001:2.0:1787828703.079505:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1787828703.079510:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:2.0:1787828703.079513:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1787828703.079514:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828703.079516:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828703.079517:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1787828703.079518:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828703.079520:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298176035 00000100:00000001:2.0:1787828703.079522:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298177280 00000100:00000001:2.0:1787828703.079524:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298176035 00000100:00000001:2.0:1787828703.079525:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298177280 00000100:00000001:2.0:1787828703.079526:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787828703.079528:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828703.079529:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1787828703.079531:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828703/real 1787828703] req@ffff8d30d13ec780 x1874670646756608/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828728 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1787828703.079535:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787828703.079536:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1787828703.079537:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1787828703.079538:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787828703.079539:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828703.079540:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828703.079540:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787828703.079543:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d13ec780 x1874670646756608/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828728 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787828703.079546:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828703.079548:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787828703.079549:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828703.079550:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828703.079551:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828703.079552:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787828703.079554:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787828703.079555:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828703.079557:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:2.0:1787828703.079558:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1787828703.079560:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d13ec780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646756608:192.168.203.143@tcp:8:kworker.0 00000100:00000001:2.0:1787828703.079562:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828703.079564:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828703.079564:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787828703.079565:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1787828703.079567:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787828703.079568:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:2.0:1787828703.079568:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:2.0:1787828703.079570:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787828703.079570:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1787828703.079572:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787828703.079572:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828703.079574:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828703.079574:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787828703.079575:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1787828703.079576:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787828703.079577:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:2.0:1787828703.079578:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:2.0:1787828703.079578:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787828703.079579:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1787828703.079580:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787828703.079581:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828703.079582:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828703.079582:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787828703.079583:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1787828703.079584:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787828703.079584:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:2.0:1787828703.079585:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:2.0:1787828703.079586:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787828703.079587:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1787828703.079588:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787828703.079588:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828703.079589:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828703.079591:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1787828703.079592:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828703.079593:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787828703.079594:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828703.974702:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828703.974707:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828703.974753:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828703.974757:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828703.974760:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828704.998726:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828704.998733:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828704.998821:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828704.998825:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828704.998829:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828706.022696:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828706.022701:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828706.022755:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828706.022759:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828706.022763:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828707.046718:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828707.046725:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828707.046778:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828707.046781:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828707.046784:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828708.070703:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828708.070711:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828708.070777:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828708.070780:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828708.070783:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787828708.198712:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828708.198723:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1787828708.198726:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787828708.198729:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787828708.198734:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787828708.198741:0:40701:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 3508 00000100:00080000:2.0:1787828708.198752:0:40701:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787828708.198756:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828708.198757:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787828708.198761:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787828708.198762:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787828708.198765:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:2.0:1787828708.198767:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:2.0:1787828708.198768:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828708.198769:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:2.0:1787828708.198773:0:40701:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787828708.198775:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828708.198777:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787828708.198780:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828708.198782:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828708.198791:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:2.0:1787828708.198793:0:40701:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828708.198817:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787828708.198818:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828708.198821:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787828708.198823:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787828708.198827:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828708.198836:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787828708.198844:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2da40 x1874670646756736/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787828708.198852:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787828708.198855:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828708.198868:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787828708.198871:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828708.198874:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1787828708.198875:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787828708.198876:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787828708.198878:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787828708.198880:0:40701:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3508 00000100:00080000:2.0:1787828708.198882:0:40701:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787828708.198885:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828708.198886:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787828708.198888:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787828708.198889:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787828708.198891:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000020:00000001:2.0:1787828708.198893:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:2.0:1787828708.198894:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828708.198896:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:2.0:1787828708.198898:0:40701:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787828708.198900:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828708.198902:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:2.0:1787828708.198902:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1787828708.198907:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 02000000:00000001:2.0:1787828708.198908:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787828708.198910:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 02000000:00000001:2.0:1787828708.198912:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828708.198914:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787828708.198915:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:1.0:1787828708.198916:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000008:00100000:2.0:1787828708.198921:0:40701:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 02000000:00000001:1.0:1787828708.198922:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787828708.198926:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000008:00000001:2.0:1787828708.198928:0:40701:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828708.198931:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828708.198933:0:40701:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828708.198939:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00100000:1.0:1787828708.198940:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2da40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646756736:192.168.203.143@tcp:250:kworker.0 02000000:00000001:2.0:1787828708.198942:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828708.198948:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:1.0:1787828708.198961:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787828708.198963:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1787828708.198967:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1787828708.198968:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828708.198973:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828708.198976:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828708.198977:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787828708.198985:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787828708.198986:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2ed00 x1874670646756864/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787828708.198994:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787828708.198996:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828708.198999:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787828708.199001:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1787828708.199004:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00080000:2.0:1787828708.199004:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:1.0:1787828708.199008:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417875008 : -126234291676608 : ffff8d30c9e2da40) 00000100:00000001:2.0:1787828708.199009:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787828708.199011:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787828708.199025:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:1.0:1787828708.199026:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00080000:2.0:1787828708.199028:0:40701:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3508 00000100:00080000:2.0:1787828708.199033:0:40701:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787828708.199035:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828708.199036:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787828708.199038:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787828708.199039:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787828708.199040:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000020:00000001:2.0:1787828708.199042:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:2.0:1787828708.199043:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828708.199044:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:2.0:1787828708.199046:0:40701:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787828708.199048:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828708.199049:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787828708.199052:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828708.199055:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828708.199056:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1787828708.199058:0:40701:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1787828708.199059:0:40701:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828708.199061:0:40701:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828708.199066:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000800:00000100:1.0:1787828708.199069:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 02000000:00000001:2.0:1787828708.199069:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828708.199072:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787828708.199074:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787828708.199076:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828708.199080:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828708.199085:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00100000:2.0:1787828708.199086:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2c000 x1874670646756992/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787828708.199092:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787828708.199095:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828708.199099:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787828708.199100:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1787828708.199106:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828708.199109:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828708.199110:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828708.199116:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4298182400 00000100:00080000:2.0:1787828708.199116:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828708.199124:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1787828708.199140:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787828708.199141:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787828708.199154:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787828708.199156:0:40701:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3508 00000100:00000001:1.0:1787828708.199159:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 00000100:00080000:2.0:1787828708.199161:0:40701:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787828708.199164:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828708.199166:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787828708.199168:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787828708.199169:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787828708.199170:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000020:00000001:2.0:1787828708.199172:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:2.0:1787828708.199173:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828708.199174:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:1.0:1787828708.199177:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4298182400 00000100:00080000:2.0:1787828708.199177:0:40701:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:1.0:1787828708.199186:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 00000100:00000001:2.0:1787828708.199186:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828708.199191:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1787828708.199194:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828708.199195:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787828708.199199:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828708.199200:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828708.199205:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1787828708.199218:0:40701:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000008:00000001:2.0:1787828708.199219:0:40701:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828708.199220:0:40701:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828708.199223:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787828708.199224:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828708.199225:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787828708.199227:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:1.0:1787828708.199228:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 02000000:00000001:2.0:1787828708.199229:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828708.199236:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787828708.199245:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2f480 x1874670646757120/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787828708.199246:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828708/real 1787828708] req@ffff8d30c9e2da40 x1874670646756736/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828733 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787828708.199261:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787828708.199265:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828708.199269:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:1.0:1787828708.199273:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787828708.199277:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787828708.199281:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787828708.199284:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787828708.199287:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828708.199290:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787828708.199292:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787828708.199299:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2da40 x1874670646756736/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828733 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787828708.199308:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828708.199312:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787828708.199315:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828708.199317:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787828708.199318:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828708.199320:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787828708.199322:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787828708.199325:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828708.199329:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787828708.199332:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787828708.199336:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2da40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646756736:192.168.203.143@tcp:250:kworker.0 00000100:00000001:1.0:1787828708.199357:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828708.199359:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787828708.199361:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787828708.199364:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787828708.199366:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787828708.199369:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787828708.199370:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787828708.199374:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787828708.199376:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787828708.199379:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787828708.199381:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828708.199383:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828708.199391:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787828708.199392:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828708.199395:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787828708.199397:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1787828708.199398:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787828708.199400:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787828708.199402:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828708.199404:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787828708.199405:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787828708.199408:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787828708.199411:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646756864:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787828708.199415:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787828708.199417:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787828708.199425:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828708.199427:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787828708.199430:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828708.199436:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787828708.199438:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417879808 : -126234291671808 : ffff8d30c9e2ed00) 00000100:00000001:1.0:1787828708.199442:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1787828708.199457:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787828708.199462:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787828708.199464:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787828708.199466:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828708.199467:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787828708.199470:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828708.199475:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298181142 00000100:00000001:1.0:1787828708.199478:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298182400 00000100:00000001:1.0:1787828708.199480:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298181142 00000100:00000001:1.0:1787828708.199482:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298182400 00000100:00000001:1.0:1787828708.199484:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787828708.199486:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828708.199488:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787828708.199493:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828708/real 1787828708] req@ffff8d30c9e2ed00 x1874670646756864/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828733 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787828708.199502:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787828708.199503:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787828708.199506:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787828708.199508:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787828708.199509:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828708.199511:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787828708.199512:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787828708.199516:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2ed00 x1874670646756864/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828733 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787828708.199523:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828708.199526:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787828708.199529:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828708.199530:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787828708.199532:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828708.199533:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787828708.199535:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787828708.199538:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828708.199541:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787828708.199544:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787828708.199547:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646756864:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787828708.199551:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787828708.199552:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787828708.199554:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828708.199556:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787828708.199558:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787828708.199560:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787828708.199563:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646756992:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787828708.199566:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787828708.199568:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787828708.199570:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828708.199572:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787828708.199575:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828708.199581:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787828708.199582:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417868288 : -126234291683328 : ffff8d30c9e2c000) 00000100:00000001:1.0:1787828708.199585:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1787828708.199597:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787828708.199621:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787828708.199623:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787828708.199625:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828708.199627:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787828708.199629:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828708.199633:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298181143 00000100:00000001:1.0:1787828708.199636:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298182400 00000100:00000001:1.0:1787828708.199639:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298181143 00000100:00000001:1.0:1787828708.199641:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298182400 00000100:00000001:1.0:1787828708.199643:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787828708.199645:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828708.199647:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787828708.199651:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828708/real 1787828708] req@ffff8d30c9e2c000 x1874670646756992/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828733 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787828708.199660:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787828708.199661:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787828708.199663:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787828708.199665:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787828708.199666:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828708.199668:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787828708.199669:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787828708.199673:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2c000 x1874670646756992/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828733 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787828708.199680:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828708.199683:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787828708.199685:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828708.199687:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787828708.199688:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828708.199690:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787828708.199692:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787828708.199694:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828708.199698:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787828708.199700:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787828708.199703:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646756992:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787828708.199707:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787828708.199708:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787828708.199710:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828708.199712:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787828708.199714:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787828708.199716:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787828708.199719:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646757120:192.168.203.143@tcp:8:kworker.0 00000100:00000001:1.0:1787828708.199723:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787828708.199724:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787828708.199726:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828708.199728:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787828708.199731:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828708.199736:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787828708.199737:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417881728 : -126234291669888 : ffff8d30c9e2f480) 00000100:00000001:1.0:1787828708.199740:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1787828708.199751:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787828708.199755:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787828708.199758:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787828708.199759:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828708.199761:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787828708.199763:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828708.199766:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298181156 00000100:00000001:1.0:1787828708.199769:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298182400 00000100:00000001:1.0:1787828708.199771:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298181156 00000100:00000001:1.0:1787828708.199773:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298182400 00000100:00000001:1.0:1787828708.199775:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787828708.199777:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828708.199779:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787828708.199783:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828708/real 1787828708] req@ffff8d30c9e2f480 x1874670646757120/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828733 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787828708.199790:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787828708.199792:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787828708.199794:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787828708.199796:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787828708.199797:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828708.199799:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787828708.199801:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787828708.199804:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2f480 x1874670646757120/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828733 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787828708.199821:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828708.199823:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787828708.199825:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828708.199827:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787828708.199828:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828708.199830:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787828708.199832:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787828708.199834:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828708.199837:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787828708.199839:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787828708.199842:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646757120:192.168.203.143@tcp:8:kworker.0 00000100:00000001:1.0:1787828708.199846:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828708.199847:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787828708.199849:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787828708.199850:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787828708.199852:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787828708.199854:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787828708.199855:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787828708.199857:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787828708.199859:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787828708.199863:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787828708.199865:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828708.199867:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787828708.199868:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787828708.199869:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787828708.199881:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787828708.199883:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787828708.199884:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787828708.199885:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787828708.199886:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787828708.199889:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787828708.199890:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828708.199892:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787828708.199893:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787828708.199894:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787828708.199896:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787828708.199897:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787828708.199898:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787828708.199900:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787828708.199901:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787828708.199903:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787828708.199904:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828708.199906:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828708.199909:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787828708.199911:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828708.199913:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787828708.199914:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828709.094694:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828709.094699:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828709.094754:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828709.094758:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828709.094762:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828710.118702:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828710.118708:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828710.118775:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828710.118779:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828710.118783:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828711.142745:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828711.142752:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828711.142812:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828711.142816:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828711.142819:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828712.166765:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828712.166770:0:815:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1787828712.166772:0:815:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1787828712.166775:0:815:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828712.166785:0:815:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1787828712.166787:0:815:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828712.166789:0:815:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828712.166790:0:815:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828712.166850:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828712.166854:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828712.166856:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828713.190761:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828713.190769:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828713.190835:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828713.190839:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828713.190842:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787828713.318734:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828713.318744:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1787828713.318747:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1787828713.318751:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787828713.318755:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1787828713.318763:0:40701:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 3513 00000100:00080000:1.0:1787828713.318775:0:40701:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1787828713.318777:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787828713.318779:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:1.0:1787828713.318783:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:1.0:1787828713.318786:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:1.0:1787828713.318789:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:1.0:1787828713.318790:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:1.0:1787828713.318792:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787828713.318793:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:1.0:1787828713.318797:0:40701:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:1.0:1787828713.318799:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828713.318801:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1787828713.318805:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787828713.318808:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828713.318810:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:1.0:1787828713.318812:0:40701:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828713.318826:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1787828713.318828:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828713.318831:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1787828713.318834:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1787828713.318838:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828713.318850:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787828713.318860:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d13ee940 x1874670646757248/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787828713.318869:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787828713.318873:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828713.318896:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1787828713.318898:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828713.318901:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1787828713.318902:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1787828713.318903:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787828713.318905:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1787828713.318907:0:40701:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3513 00000100:00080000:1.0:1787828713.318909:0:40701:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1787828713.318910:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787828713.318911:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:1.0:1787828713.318913:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:1.0:1787828713.318914:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:1.0:1787828713.318915:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000020:00000001:1.0:1787828713.318917:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:1.0:1787828713.318918:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787828713.318920:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:1.0:1787828713.318922:0:40701:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:1.0:1787828713.318923:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828713.318925:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1787828713.318931:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787828713.318933:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828713.318935:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:1.0:1787828713.318939:0:40701:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1787828713.318942:0:40701:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828713.318943:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787828713.318944:0:40701:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828713.318951:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 02000000:00000001:1.0:1787828713.318952:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1787828713.318955:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828713.318955:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787828713.318960:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:2.0:1787828713.318960:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 02000000:00000001:1.0:1787828713.318962:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:2.0:1787828713.318963:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828713.318964:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828713.318970:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:1.0:1787828713.318971:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828713.318973:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828713.318977:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787828713.318978:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d13ec780 x1874670646757376/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787828713.318984:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d13ee940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646757248:192.168.203.143@tcp:250:kworker.0 00000100:00000001:1.0:1787828713.318990:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787828713.318992:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:1.0:1787828713.318993:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828713.318997:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828713.318998:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000100:00080000:1.0:1787828713.319000:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 02000000:00000001:2.0:1787828713.319001:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1787828713.319007:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 02000000:00000001:2.0:1787828713.319008:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:1.0:1787828713.319010:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1787828713.319012:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787828713.319013:0:40701:0:(import.c:485:import_select_connection()) Process entered 02000000:00000001:2.0:1787828713.319015:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1787828713.319017:0:40701:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3513 00000100:00080000:1.0:1787828713.319020:0:40701:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1787828713.319022:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787828713.319023:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:1.0:1787828713.319025:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:1.0:1787828713.319026:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:1.0:1787828713.319027:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000020:00000001:1.0:1787828713.319028:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:1.0:1787828713.319030:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787828713.319032:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:2.0:1787828713.319034:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00080000:1.0:1787828713.319035:0:40701:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787828713.319038:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839541348672 : -126234168202944 : ffff8d30d13ee940) 00000100:00000001:1.0:1787828713.319039:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828713.319043:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:2.0:1787828713.319044:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 02000000:00000001:1.0:1787828713.319046:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787828713.319049:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828713.319050:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:1.0:1787828713.319051:0:40701:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1787828713.319053:0:40701:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828713.319054:0:40701:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828713.319057:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1787828713.319058:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828713.319059:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1787828713.319060:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1787828713.319061:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828713.319065:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787828713.319068:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d13ec000 x1874670646757504/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787828713.319074:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787828713.319075:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828713.319077:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1787828713.319079:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828713.319081:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1787828713.319082:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1787828713.319083:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787828713.319084:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1787828713.319088:0:40701:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3513 00000800:00000100:2.0:1787828713.319104:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00080000:1.0:1787828713.319105:0:40701:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1787828713.319108:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787828713.319109:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:1.0:1787828713.319111:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:1.0:1787828713.319113:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:1.0:1787828713.319114:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000020:00000001:1.0:1787828713.319116:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:1.0:1787828713.319118:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787828713.319119:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:2.0:1787828713.319123:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00080000:1.0:1787828713.319124:0:40701:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:1.0:1787828713.319128:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828713.319129:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:2.0:1787828713.319130:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 02000000:00000001:1.0:1787828713.319132:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787828713.319134:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828713.319135:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828713.319140:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:2.0:1787828713.319140:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000008:00100000:1.0:1787828713.319144:0:40701:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000008:00000001:1.0:1787828713.319146:0:40701:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828713.319147:0:40701:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828713.319148:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828713.319152:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:2.0:1787828713.319154:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4298187520 02000000:00000001:1.0:1787828713.319155:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828713.319159:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1787828713.319160:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1787828713.319162:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828713.319166:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828713.319167:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 00000100:00100000:1.0:1787828713.319171:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d13ecb40 x1874670646757632/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787828713.319178:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787828713.319179:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828713.319180:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828713.319187:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4298187520 00000100:00000001:2.0:1787828713.319195:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 00000100:00000001:2.0:1787828713.319200:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787828713.319204:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828713.319207:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1787828713.319216:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828713/real 1787828713] req@ffff8d30d13ee940 x1874670646757248/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828738 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1787828713.319228:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787828713.319230:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1787828713.319235:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1787828713.319238:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787828713.319240:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828713.319243:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828713.319245:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787828713.319251:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d13ee940 x1874670646757248/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828738 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787828713.319260:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828713.319264:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787828713.319268:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828713.319270:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828713.319271:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828713.319274:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787828713.319276:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787828713.319278:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828713.319282:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:2.0:1787828713.319284:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1787828713.319287:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d13ee940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646757248:192.168.203.143@tcp:250:kworker.0 00000100:00000001:2.0:1787828713.319291:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828713.319293:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828713.319294:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787828713.319297:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1787828713.319300:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787828713.319302:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:2.0:1787828713.319304:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:2.0:1787828713.319309:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787828713.319310:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1787828713.319313:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787828713.319314:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828713.319316:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828713.319323:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1787828713.319325:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828713.319326:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787828713.319328:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1787828713.319329:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1787828713.319330:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787828713.319332:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828713.319333:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787828713.319335:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828713.319336:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787828713.319338:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d13ec780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646757376:192.168.203.143@tcp:38:kworker.0 00000100:00000001:2.0:1787828713.319341:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787828713.319342:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1787828713.319344:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828713.319345:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1787828713.319347:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828713.319353:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787828713.319354:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839541340032 : -126234168211584 : ffff8d30d13ec780) 00000100:00000001:2.0:1787828713.319365:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1787828713.319376:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:2.0:1787828713.319380:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1787828713.319381:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828713.319382:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828713.319384:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1787828713.319386:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828713.319390:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298186262 00000100:00000001:2.0:1787828713.319392:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298187520 00000100:00000001:2.0:1787828713.319394:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298186262 00000100:00000001:2.0:1787828713.319396:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298187520 00000100:00000001:2.0:1787828713.319398:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787828713.319399:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828713.319400:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1787828713.319403:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828713/real 1787828713] req@ffff8d30d13ec780 x1874670646757376/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828738 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1787828713.319408:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787828713.319409:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1787828713.319427:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1787828713.319428:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787828713.319436:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828713.319438:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828713.319439:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787828713.319442:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d13ec780 x1874670646757376/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828738 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787828713.319448:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828713.319449:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787828713.319451:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828713.319452:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828713.319453:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828713.319454:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787828713.319455:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787828713.319457:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828713.319459:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:2.0:1787828713.319461:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1787828713.319463:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d13ec780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646757376:192.168.203.143@tcp:38:kworker.0 00000100:00000001:2.0:1787828713.319466:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1787828713.319467:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787828713.319468:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828713.319469:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787828713.319470:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828713.319472:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787828713.319473:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d13ec000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646757504:192.168.203.143@tcp:38:kworker.0 00000100:00000001:2.0:1787828713.319476:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787828713.319477:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1787828713.319479:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828713.319480:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1787828713.319483:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828713.319486:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787828713.319488:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839541338112 : -126234168213504 : ffff8d30d13ec000) 00000100:00000001:2.0:1787828713.319491:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1787828713.319499:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:2.0:1787828713.319502:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1787828713.319504:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828713.319505:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828713.319506:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1787828713.319508:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828713.319511:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298186262 00000100:00000001:2.0:1787828713.319513:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298187520 00000100:00000001:2.0:1787828713.319514:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298186262 00000100:00000001:2.0:1787828713.319516:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298187520 00000100:00000001:2.0:1787828713.319518:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787828713.319519:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828713.319521:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1787828713.319523:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828713/real 1787828713] req@ffff8d30d13ec000 x1874670646757504/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828738 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1787828713.319528:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787828713.319529:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1787828713.319530:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1787828713.319531:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787828713.319532:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828713.319533:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828713.319534:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787828713.319536:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d13ec000 x1874670646757504/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828738 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787828713.319541:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828713.319543:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787828713.319545:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828713.319546:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828713.319547:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828713.319548:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787828713.319549:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787828713.319550:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828713.319553:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:2.0:1787828713.319555:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1787828713.319557:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d13ec000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646757504:192.168.203.143@tcp:38:kworker.0 00000100:00000001:2.0:1787828713.319560:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1787828713.319561:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787828713.319562:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828713.319563:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787828713.319564:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828713.319565:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787828713.319567:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d13ecb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646757632:192.168.203.143@tcp:8:kworker.0 00000100:00000001:2.0:1787828713.319569:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787828713.319570:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1787828713.319571:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828713.319572:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1787828713.319574:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828713.319577:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787828713.319578:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839541340992 : -126234168210624 : ffff8d30d13ecb40) 00000100:00000001:2.0:1787828713.319580:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1787828713.319586:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:2.0:1787828713.319589:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1787828713.319591:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828713.319592:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828713.319593:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1787828713.319594:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828713.319597:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298186275 00000100:00000001:2.0:1787828713.319631:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298187520 00000100:00000001:2.0:1787828713.319635:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298186276 00000100:00000001:2.0:1787828713.319637:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298187520 00000100:00000001:2.0:1787828713.319640:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787828713.319658:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828713.319660:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1787828713.319666:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828713/real 1787828713] req@ffff8d30d13ecb40 x1874670646757632/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828738 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1787828713.319675:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787828713.319676:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1787828713.319679:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1787828713.319681:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787828713.319683:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828713.319685:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828713.319686:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787828713.319690:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d13ecb40 x1874670646757632/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828738 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787828713.319695:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828713.319696:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787828713.319704:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828713.319705:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828713.319706:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828713.319708:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787828713.319709:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787828713.319711:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828713.319713:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:2.0:1787828713.319715:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1787828713.319716:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d13ecb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646757632:192.168.203.143@tcp:8:kworker.0 00000100:00000001:2.0:1787828713.319719:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828713.319720:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828713.319721:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787828713.319722:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1787828713.319723:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787828713.319725:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:2.0:1787828713.319726:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:2.0:1787828713.319727:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787828713.319728:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1787828713.319730:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787828713.319732:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828713.319733:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828713.319734:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787828713.319735:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1787828713.319737:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787828713.319737:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:2.0:1787828713.319738:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:2.0:1787828713.319739:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787828713.319740:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1787828713.319741:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787828713.319742:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828713.319743:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828713.319743:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787828713.319744:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1787828713.319745:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787828713.319746:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:2.0:1787828713.319746:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:2.0:1787828713.319747:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787828713.319748:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1787828713.319749:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787828713.319749:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828713.319751:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828713.319753:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1787828713.319753:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828713.319755:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787828713.319756:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828714.214744:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828714.214751:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828714.214802:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828714.214807:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828714.214809:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828715.238723:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828715.238729:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828715.238850:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828715.238855:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828715.238859:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828716.262693:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828716.262699:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828716.262750:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828716.262753:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828716.262756:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828717.286716:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828717.286723:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828717.286778:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828717.286781:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828717.286784:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828718.310676:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828718.310681:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828718.310732:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828718.310736:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828718.310739:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787828718.438674:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828718.438685:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1787828718.438688:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1787828718.438693:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787828718.438697:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1787828718.438704:0:40701:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 3518 00000100:00080000:1.0:1787828718.438714:0:40701:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1787828718.438719:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787828718.438722:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:1.0:1787828718.438726:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:1.0:1787828718.438729:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:1.0:1787828718.438733:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:1.0:1787828718.438736:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:1.0:1787828718.438739:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787828718.438741:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:1.0:1787828718.438746:0:40701:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:1.0:1787828718.438750:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828718.438754:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1787828718.438759:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787828718.438763:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828718.438766:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:1.0:1787828718.438769:0:40701:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828718.438788:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1787828718.438791:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828718.438795:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1787828718.438798:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1787828718.438802:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828718.438814:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787828718.438825:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d13ec3c0 x1874670646757760/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787828718.438836:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787828718.438840:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828718.438864:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1787828718.438868:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828718.438873:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1787828718.438875:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1787828718.438878:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787828718.438882:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1787828718.438886:0:40701:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3518 00000100:00000001:0.0:1787828718.438888:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00080000:1.0:1787828718.438891:0:40701:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787828718.438894:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1787828718.438895:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787828718.438898:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787828718.438899:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:0.0:1787828718.438901:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787828718.438904:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787828718.438904:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:1.0:1787828718.438907:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:1.0:1787828718.438910:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 02000000:00000001:0.0:1787828718.438911:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787828718.438916:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000020:00000001:1.0:1787828718.438916:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:1.0:1787828718.438920:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 02000000:00000001:0.0:1787828718.438923:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828718.438923:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:1.0:1787828718.438934:0:40701:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00100000:0.0:1787828718.438935:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d13ec3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646757760:192.168.203.143@tcp:250:kworker.0 00000100:00000001:1.0:1787828718.438938:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828718.438942:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:0.0:1787828718.438945:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787828718.438949:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787828718.438951:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787828718.438953:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828718.438956:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828718.438957:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 02000000:00000001:0.0:1787828718.438962:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000008:00100000:1.0:1787828718.438963:0:40701:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1787828718.438967:0:40701:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828718.438969:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828718.438970:0:40701:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828718.438977:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1787828718.438979:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828718.438981:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1787828718.438983:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1787828718.438985:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828718.438987:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787828718.438991:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839541339072 : -126234168212544 : ffff8d30d13ec3c0) 00000100:00000001:1.0:1787828718.438992:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828718.438997:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00100000:1.0:1787828718.439001:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d13ed680 x1874670646757888/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787828718.439011:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787828718.439013:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828718.439017:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1787828718.439020:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828718.439024:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:1.0:1787828718.439026:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1787828718.439028:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787828718.439031:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1787828718.439034:0:40701:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3518 00000100:00080000:1.0:1787828718.439039:0:40701:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1787828718.439041:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787828718.439043:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:1.0:1787828718.439046:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000800:00000100:0.0:1787828718.439049:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000020:00000001:1.0:1787828718.439050:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:1.0:1787828718.439054:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000020:00000001:1.0:1787828718.439073:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:0.0:1787828718.439078:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787828718.439080:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787828718.439083:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:0.0:1787828718.439085:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828718.439090:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1787828718.439090:0:40701:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787828718.439095:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787828718.439097:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828718.439101:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:0.0:1787828718.439103:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828718.439105:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787828718.439110:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828718.439111:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4298192640 00000100:00000001:1.0:1787828718.439113:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:1.0:1787828718.439117:0:40701:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1787828718.439120:0:40701:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828718.439121:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 00000100:00000001:1.0:1787828718.439123:0:40701:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828718.439130:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4298192640 02000000:00000001:1.0:1787828718.439131:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1787828718.439134:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828718.439137:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:0.0:1787828718.439139:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 02000000:00000001:1.0:1787828718.439139:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1787828718.439141:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828718.439146:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787828718.439149:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828718.439151:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828718.439156:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787828718.439158:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d13ed2c0 x1874670646758016/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787828718.439165:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828718/real 1787828718] req@ffff8d30d13ec3c0 x1874670646757760/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828743 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787828718.439175:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787828718.439181:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828718.439186:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:0.0:1787828718.439187:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00080000:1.0:1787828718.439190:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000001:0.0:1787828718.439191:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00080000:1.0:1787828718.439197:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1787828718.439198:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787828718.439199:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:0.0:1787828718.439204:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00080000:1.0:1787828718.439204:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787828718.439210:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828718.439210:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:0.0:1787828718.439216:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00080000:1.0:1787828718.439217:0:40701:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3518 00000100:00000001:0.0:1787828718.439220:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:1.0:1787828718.439224:0:40701:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1787828718.439228:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00100000:0.0:1787828718.439231:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d13ec3c0 x1874670646757760/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828743 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787828718.439231:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:1.0:1787828718.439238:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:1.0:1787828718.439240:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:1.0:1787828718.439244:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:0.0:1787828718.439249:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:1.0:1787828718.439250:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:0.0:1787828718.439256:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787828718.439256:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787828718.439259:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:0.0:1787828718.439263:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:1.0:1787828718.439267:0:40701:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787828718.439268:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828718.439272:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828718.439273:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828718.439279:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 02000000:00000001:1.0:1787828718.439279:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:0.0:1787828718.439284:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 02000000:00000001:1.0:1787828718.439285:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00080000:0.0:1787828718.439291:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 02000000:00000001:1.0:1787828718.439291:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828718.439298:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00080000:0.0:1787828718.439301:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000008:00100000:1.0:1787828718.439301:0:40701:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000008:00000001:1.0:1787828718.439306:0:40701:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828718.439308:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000001:1.0:1787828718.439311:0:40701:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828718.439317:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d13ec3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646757760:192.168.203.143@tcp:250:kworker.0 02000000:00000001:1.0:1787828718.439317:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1787828718.439321:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828718.439326:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:0.0:1787828718.439329:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:1.0:1787828718.439329:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1787828718.439333:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828718.439334:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828718.439338:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 00000100:00000001:1.0:1787828718.439340:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828718.439341:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787828718.439347:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787828718.439351:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000100:00100000:1.0:1787828718.439351:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d13efc00 x1874670646758144/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000020:00000001:0.0:1787828718.439355:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787828718.439360:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787828718.439363:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787828718.439367:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787828718.439368:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787828718.439371:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828718.439372:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828718.439377:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828718.439378:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828718.439388:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787828718.439390:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828718.439393:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787828718.439395:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787828718.439397:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1787828718.439399:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787828718.439400:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828718.439403:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787828718.439406:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787828718.439409:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828718.439413:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d13ed680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646757888:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787828718.439417:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787828718.439420:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787828718.439422:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828718.439424:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1787828718.439428:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828718.439437:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787828718.439439:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839541343872 : -126234168207744 : ffff8d30d13ed680) 00000100:00000001:0.0:1787828718.439444:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1787828718.439461:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:0.0:1787828718.439468:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1787828718.439472:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828718.439474:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828718.439477:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787828718.439481:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828718.439486:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298191382 00000100:00000001:0.0:1787828718.439490:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298192640 00000100:00000001:0.0:1787828718.439493:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298191382 00000100:00000001:0.0:1787828718.439496:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298192640 00000100:00000001:0.0:1787828718.439499:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787828718.439513:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828718.439516:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787828718.439525:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828718/real 1787828718] req@ffff8d30d13ed680 x1874670646757888/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828743 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787828718.439535:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787828718.439537:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787828718.439540:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787828718.439543:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787828718.439544:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828718.439547:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828718.439548:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787828718.439552:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d13ed680 x1874670646757888/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828743 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787828718.439561:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828718.439563:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787828718.439566:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828718.439568:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828718.439569:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828718.439571:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787828718.439574:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787828718.439577:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828718.439581:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787828718.439583:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787828718.439587:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d13ed680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646757888:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787828718.439591:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1787828718.439593:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787828718.439594:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828718.439596:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787828718.439625:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787828718.439629:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828718.439634:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d13ed2c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646758016:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787828718.439638:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787828718.439641:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787828718.439643:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828718.439646:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1787828718.439650:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828718.439656:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787828718.439658:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839541342912 : -126234168208704 : ffff8d30d13ed2c0) 00000100:00000001:0.0:1787828718.439662:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1787828718.439677:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:0.0:1787828718.439682:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1787828718.439685:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828718.439687:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828718.439689:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787828718.439692:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828718.439695:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298191383 00000100:00000001:0.0:1787828718.439698:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298192640 00000100:00000001:0.0:1787828718.439701:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298191383 00000100:00000001:0.0:1787828718.439704:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298192640 00000100:00000001:0.0:1787828718.439706:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787828718.439709:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828718.439711:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787828718.439715:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828718/real 1787828718] req@ffff8d30d13ed2c0 x1874670646758016/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828743 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787828718.439724:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787828718.439726:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787828718.439728:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787828718.439730:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787828718.439731:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828718.439733:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828718.439735:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787828718.439739:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d13ed2c0 x1874670646758016/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828743 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787828718.439747:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828718.439749:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787828718.439752:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828718.439753:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828718.439755:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828718.439756:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787828718.439758:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787828718.439760:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828718.439764:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787828718.439767:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787828718.439770:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d13ed2c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646758016:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787828718.439775:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1787828718.439776:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787828718.439778:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828718.439780:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787828718.439782:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787828718.439784:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828718.439787:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d13efc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646758144:192.168.203.143@tcp:8:kworker.0 00000100:00000001:0.0:1787828718.439791:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787828718.439793:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787828718.439794:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828718.439796:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1787828718.439799:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828718.439805:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787828718.439806:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839541353472 : -126234168198144 : ffff8d30d13efc00) 00000100:00000001:0.0:1787828718.439810:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1787828718.439822:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:0.0:1787828718.439827:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1787828718.439830:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828718.439831:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828718.439833:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787828718.439836:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828718.439839:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298191396 00000100:00000001:0.0:1787828718.439842:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298192640 00000100:00000001:0.0:1787828718.439844:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298191396 00000100:00000001:0.0:1787828718.439846:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298192640 00000100:00000001:0.0:1787828718.439849:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787828718.439851:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828718.439854:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787828718.439858:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828718/real 1787828718] req@ffff8d30d13efc00 x1874670646758144/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828743 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787828718.439866:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787828718.439868:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787828718.439870:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787828718.439872:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787828718.439873:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828718.439875:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828718.439877:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787828718.439880:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d13efc00 x1874670646758144/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828743 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787828718.439894:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828718.439896:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787828718.439899:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828718.439900:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828718.439902:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828718.439904:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787828718.439905:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787828718.439908:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828718.439911:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787828718.439914:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787828718.439917:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d13efc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646758144:192.168.203.143@tcp:8:kworker.0 00000100:00000001:0.0:1787828718.439921:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828718.439923:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828718.439925:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787828718.439926:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787828718.439929:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787828718.439930:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787828718.439932:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787828718.439935:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787828718.439936:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787828718.439939:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787828718.439940:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828718.439942:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828718.439943:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787828718.439945:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787828718.439946:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787828718.439948:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787828718.439949:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787828718.439951:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787828718.439953:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787828718.439954:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787828718.439955:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828718.439956:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828718.439958:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787828718.439959:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787828718.439960:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787828718.439962:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787828718.439963:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787828718.439965:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787828718.439966:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787828718.439968:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787828718.439969:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828718.439971:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828718.439975:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787828718.439977:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828718.439979:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787828718.439980:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828719.334715:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828719.334721:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828719.334776:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828719.334779:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828719.334782:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828720.358697:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828720.358703:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828720.358768:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828720.358773:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828720.358777:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828721.382737:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828721.382747:0:815:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1787828721.382749:0:815:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1787828721.382753:0:815:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828721.382758:0:815:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1787828721.382760:0:815:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828721.382763:0:815:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828721.382766:0:815:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828721.382828:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828721.382834:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828721.382840:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828722.406725:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828722.406731:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828722.406796:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828722.406801:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828722.406804:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828723.430699:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828723.430708:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828723.430777:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828723.430782:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828723.430787:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787828723.558729:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828723.558737:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1787828723.558739:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787828723.558742:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787828723.558746:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787828723.558751:0:40798:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 3524 00000100:00080000:3.0:1787828723.558759:0:40798:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787828723.558761:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787828723.558762:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:3.0:1787828723.558766:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1787828723.558785:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1787828723.558787:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:3.0:1787828723.558789:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:3.0:1787828723.558791:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787828723.558792:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:3.0:1787828723.558796:0:40798:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:3.0:1787828723.558798:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787828723.558800:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787828723.558803:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787828723.558805:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828723.558807:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:3.0:1787828723.558819:0:40798:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787828723.558829:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787828723.558830:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828723.558833:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787828723.558835:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787828723.558838:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828723.558848:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787828723.558855:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c7016580 x1874670646758272/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787828723.558881:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787828723.558884:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828723.558898:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787828723.558900:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828723.558902:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1787828723.558903:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787828723.558904:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787828723.558906:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787828723.558908:0:40798:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3524 00000100:00080000:3.0:1787828723.558920:0:40798:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787828723.558921:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787828723.558922:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:3.0:1787828723.558923:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1787828723.558924:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1787828723.558926:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000020:00000001:3.0:1787828723.558927:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:3.0:1787828723.558929:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787828723.558930:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:3.0:1787828723.558932:0:40798:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:3.0:1787828723.558933:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787828723.558935:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787828723.558955:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787828723.558957:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828723.558959:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:3.0:1787828723.558962:0:40798:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1787828723.558965:0:40798:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828723.558966:0:40798:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828723.558968:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:3.0:1787828723.558969:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787828723.558973:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828723.558975:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787828723.558976:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:0.0:1787828723.558977:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 02000000:00000001:3.0:1787828723.558978:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828723.558983:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787828723.558984:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828723.558989:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00100000:3.0:1787828723.558991:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c70161c0 x1874670646758400/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787828723.558994:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828723.559004:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:3.0:1787828723.559004:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787828723.559009:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828723.559011:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787828723.559012:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828723.559019:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787828723.559050:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828723.559056:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1787828723.559058:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787828723.559061:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1787828723.559062:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c7016580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646758272:192.168.203.143@tcp:250:kworker.0 00000100:00000001:3.0:1787828723.559063:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787828723.559078:0:40798:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3524 00000100:00000001:0.0:1787828723.559085:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00080000:3.0:1787828723.559085:0:40798:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787828723.559089:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787828723.559090:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 02000000:00000001:0.0:1787828723.559091:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000020:00000001:3.0:1787828723.559092:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1787828723.559095:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 02000000:00000001:0.0:1787828723.559097:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828723.559097:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000020:00000001:3.0:1787828723.559103:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 02000000:00000001:0.0:1787828723.559104:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:3.0:1787828723.559105:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787828723.559108:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:3.0:1787828723.559109:0:40798:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 02000000:00000001:0.0:1787828723.559112:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828723.559112:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787828723.559116:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787828723.559119:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787828723.559121:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828723.559122:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:3.0:1787828723.559124:0:40798:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1787828723.559125:0:40798:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828723.559127:0:40798:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787828723.559131:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:0.0:1787828723.559133:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 02000000:00000001:3.0:1787828723.559133:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828723.559137:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:0.0:1787828723.559138:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369545088 : -126234340006528 : ffff8d30c7016580) 02000000:00000001:3.0:1787828723.559139:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787828723.559141:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828723.559147:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:3.0:1787828723.559148:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787828723.559153:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c7015a40 x1874670646758528/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787828723.559158:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787828723.559160:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828723.559177:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787828723.559180:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828723.559182:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1787828723.559183:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787828723.559184:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787828723.559186:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787828723.559188:0:40798:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3524 00000100:00080000:3.0:1787828723.559191:0:40798:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787828723.559193:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787828723.559194:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:3.0:1787828723.559195:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1787828723.559196:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1787828723.559197:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000020:00000001:3.0:1787828723.559199:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:3.0:1787828723.559200:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000800:00000100:0.0:1787828723.559209:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:3.0:1787828723.559209:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:3.0:1787828723.559216:0:40798:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:3.0:1787828723.559218:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787828723.559220:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787828723.559222:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787828723.559223:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828723.559225:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1787828723.559227:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000008:00100000:3.0:1787828723.559227:0:40798:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1787828723.559231:0:40798:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828723.559233:0:40798:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828723.559234:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 02000000:00000001:3.0:1787828723.559236:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:0.0:1787828723.559241:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787828723.559241:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828723.559245:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:0.0:1787828723.559248:0:2390:0:(events.c:68:request_out_callback()) Process leaving 02000000:00000001:3.0:1787828723.559248:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787828723.559252:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828723.559257:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828723.559258:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787828723.559261:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c7014f00 x1874670646758656/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787828723.559266:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4298197760 00000100:00000001:3.0:1787828723.559268:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787828723.559272:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828723.559274:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828723.559281:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1787828723.559290:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4298197760 00000100:00000001:0.0:1787828723.559297:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1787828723.559301:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787828723.559304:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828723.559307:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787828723.559314:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828723/real 1787828723] req@ffff8d30c7016580 x1874670646758272/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828748 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787828723.559327:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787828723.559329:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787828723.559332:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787828723.559335:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787828723.559337:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828723.559339:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828723.559341:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787828723.559347:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c7016580 x1874670646758272/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828748 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787828723.559356:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828723.559360:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787828723.559363:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828723.559365:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828723.559366:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828723.559369:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787828723.559371:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787828723.559375:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828723.559380:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787828723.559382:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787828723.559387:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c7016580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646758272:192.168.203.143@tcp:250:kworker.0 00000100:00000001:0.0:1787828723.559393:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828723.559396:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828723.559398:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787828723.559400:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787828723.559404:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787828723.559406:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787828723.559408:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787828723.559413:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787828723.559415:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787828723.559434:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787828723.559435:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828723.559438:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828723.559445:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787828723.559447:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828723.559450:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787828723.559452:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787828723.559453:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1787828723.559455:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787828723.559457:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828723.559459:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787828723.559461:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787828723.559463:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828723.559467:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c70161c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646758400:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787828723.559471:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787828723.559473:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787828723.559475:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828723.559477:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1787828723.559490:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828723.559497:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787828723.559498:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369544128 : -126234340007488 : ffff8d30c70161c0) 00000100:00000001:0.0:1787828723.559503:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1787828723.559518:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:0.0:1787828723.559523:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1787828723.559526:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828723.559527:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828723.559529:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787828723.559533:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828723.559538:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298196502 00000100:00000001:0.0:1787828723.559541:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298197760 00000100:00000001:0.0:1787828723.559543:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298196502 00000100:00000001:0.0:1787828723.559545:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298197760 00000100:00000001:0.0:1787828723.559548:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787828723.559550:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828723.559552:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787828723.559557:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828723/real 1787828723] req@ffff8d30c70161c0 x1874670646758400/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828748 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787828723.559567:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787828723.559568:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787828723.559571:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787828723.559573:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787828723.559574:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828723.559576:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828723.559578:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787828723.559582:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c70161c0 x1874670646758400/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828748 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787828723.559590:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828723.559593:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787828723.559595:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828723.559597:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828723.559598:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828723.559629:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787828723.559631:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787828723.559634:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828723.559639:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787828723.559641:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787828723.559645:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c70161c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646758400:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787828723.559649:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1787828723.559650:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787828723.559652:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828723.559654:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787828723.559657:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787828723.559659:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828723.559663:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c7015a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646758528:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787828723.559666:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787828723.559668:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787828723.559670:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828723.559672:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1787828723.559675:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828723.559681:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787828723.559683:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369542208 : -126234340009408 : ffff8d30c7015a40) 00000100:00000001:0.0:1787828723.559687:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1787828723.559701:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:0.0:1787828723.559706:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1787828723.559708:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828723.559710:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828723.559711:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787828723.559714:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828723.559718:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298196503 00000100:00000001:0.0:1787828723.559721:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298197760 00000100:00000001:0.0:1787828723.559723:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298196503 00000100:00000001:0.0:1787828723.559725:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298197760 00000100:00000001:0.0:1787828723.559727:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787828723.559730:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828723.559732:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787828723.559736:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828723/real 1787828723] req@ffff8d30c7015a40 x1874670646758528/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828748 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787828723.559746:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787828723.559747:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787828723.559750:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787828723.559752:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787828723.559753:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828723.559755:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828723.559757:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787828723.559761:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c7015a40 x1874670646758528/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828748 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787828723.559776:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828723.559778:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787828723.559781:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828723.559782:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828723.559784:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828723.559786:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787828723.559787:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787828723.559790:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828723.559794:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787828723.559796:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787828723.559800:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c7015a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646758528:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787828723.559804:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1787828723.559805:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787828723.559807:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828723.559809:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787828723.559812:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787828723.559814:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828723.559817:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c7014f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646758656:192.168.203.143@tcp:8:kworker.0 00000100:00000001:0.0:1787828723.559820:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787828723.559822:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787828723.559824:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828723.559826:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1787828723.559829:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828723.559834:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787828723.559836:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369539328 : -126234340012288 : ffff8d30c7014f00) 00000100:00000001:0.0:1787828723.559839:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1787828723.559861:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:0.0:1787828723.559866:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1787828723.559869:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828723.559870:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828723.559873:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787828723.559875:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828723.559879:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298196516 00000100:00000001:0.0:1787828723.559882:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298197760 00000100:00000001:0.0:1787828723.559884:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298196516 00000100:00000001:0.0:1787828723.559887:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298197760 00000100:00000001:0.0:1787828723.559889:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787828723.559891:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828723.559893:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787828723.559898:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828723/real 1787828723] req@ffff8d30c7014f00 x1874670646758656/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828748 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787828723.559908:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787828723.559909:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787828723.559912:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787828723.559923:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787828723.559924:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828723.559926:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828723.559927:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787828723.559931:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c7014f00 x1874670646758656/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828748 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787828723.559939:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828723.559941:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787828723.559943:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828723.559945:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828723.559946:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828723.559948:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787828723.559950:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787828723.559953:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828723.559956:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787828723.559959:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787828723.559962:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c7014f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646758656:192.168.203.143@tcp:8:kworker.0 00000100:00000001:0.0:1787828723.559966:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828723.559968:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828723.559970:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787828723.559971:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787828723.559974:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787828723.559976:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787828723.559977:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787828723.559980:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787828723.559981:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787828723.559983:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787828723.559984:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828723.559986:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828723.559987:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787828723.559989:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787828723.559991:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787828723.559992:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787828723.559993:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787828723.559995:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787828723.559996:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787828723.559998:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787828723.559999:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828723.560001:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828723.560002:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787828723.560003:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787828723.560005:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787828723.560006:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787828723.560007:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787828723.560009:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787828723.560010:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787828723.560012:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787828723.560014:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828723.560016:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828723.560035:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787828723.560038:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828723.560040:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787828723.560042:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828724.454691:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828724.454698:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828724.454743:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828724.454747:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828724.454750:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828725.478673:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828725.478680:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828725.478733:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828725.478736:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828725.478739:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828726.502716:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828726.502723:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828726.502795:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828726.502800:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828726.502804:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828727.526711:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828727.526717:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828727.526817:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828727.526821:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828727.526824:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828728.550711:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828728.550718:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828728.550793:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828728.550797:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828728.550812:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787828728.678681:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828728.678690:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1787828728.678692:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787828728.678695:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787828728.678699:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787828728.678704:0:40798:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 3529 00000100:00080000:3.0:1787828728.678722:0:40798:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787828728.678724:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787828728.678726:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:3.0:1787828728.678729:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1787828728.678731:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1787828728.678734:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:3.0:1787828728.678735:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:3.0:1787828728.678737:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787828728.678737:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:3.0:1787828728.678741:0:40798:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:3.0:1787828728.678743:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787828728.678745:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787828728.678748:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787828728.678750:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828728.678752:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:3.0:1787828728.678754:0:40798:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787828728.678763:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787828728.678764:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828728.678767:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787828728.678769:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787828728.678772:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828728.678780:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787828728.678787:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c70143c0 x1874670646758784/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787828728.678795:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787828728.678798:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828728.678811:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787828728.678813:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828728.678815:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1787828728.678816:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787828728.678818:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787828728.678820:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787828728.678822:0:40798:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3529 00000100:00080000:3.0:1787828728.678824:0:40798:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787828728.678826:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787828728.678827:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:3.0:1787828728.678828:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1787828728.678829:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1787828728.678831:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000020:00000001:3.0:1787828728.678832:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:3.0:1787828728.678850:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787828728.678851:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:3.0:1787828728.678854:0:40798:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:3.0:1787828728.678856:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787828728.678858:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787828728.678862:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787828728.678864:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828728.678866:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:3.0:1787828728.678870:0:40798:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1787828728.678873:0:40798:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828728.678874:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1787828728.678876:0:40798:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828728.678880:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 02000000:00000001:3.0:1787828728.678881:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:0.0:1787828728.678884:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 02000000:00000001:3.0:1787828728.678885:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828728.678890:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787828728.678890:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:0.0:1787828728.678895:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787828728.678895:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787828728.678899:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828728.678902:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:3.0:1787828728.678905:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828728.678906:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787828728.678911:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787828728.678914:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c7015680 x1874670646758912/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787828728.678918:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c70143c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646758784:192.168.203.143@tcp:250:kworker.0 00000100:00000001:0.0:1787828728.678927:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:3.0:1787828728.678932:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 02000000:00000001:0.0:1787828728.678933:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:3.0:1787828728.678937:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828728.678938:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828728.678941:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828728.678942:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00080000:3.0:1787828728.678947:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 02000000:00000001:0.0:1787828728.678950:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787828728.678951:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1787828728.678955:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787828728.678957:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787828728.678958:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787828728.678962:0:40798:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3529 00000100:00080000:3.0:1787828728.678965:0:40798:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787828728.678966:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787828728.678968:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787828728.678969:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:0.0:1787828728.678972:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369536448 : -126234340015168 : ffff8d30c70143c0) 00000020:00000001:3.0:1787828728.678973:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1787828728.678976:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:0.0:1787828728.678979:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000020:00000001:3.0:1787828728.678980:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000020:00000001:3.0:1787828728.678999:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:3.0:1787828728.679003:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787828728.679004:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:3.0:1787828728.679007:0:40798:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:3.0:1787828728.679009:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787828728.679010:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787828728.679013:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787828728.679015:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828728.679016:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:3.0:1787828728.679018:0:40798:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1787828728.679021:0:40798:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828728.679023:0:40798:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787828728.679043:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787828728.679044:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828728.679045:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787828728.679046:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787828728.679047:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828728.679051:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:0.0:1787828728.679056:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00100000:3.0:1787828728.679059:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c70152c0 x1874670646759040/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787828728.679066:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787828728.679069:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787828728.679069:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828728.679074:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828728.679076:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00080000:3.0:1787828728.679077:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1787828728.679081:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787828728.679082:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1787828728.679085:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1787828728.679086:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787828728.679089:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787828728.679091:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:0.0:1787828728.679093:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787828728.679095:0:40798:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3529 00000100:00000001:0.0:1787828728.679098:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4298202880 00000100:00080000:3.0:1787828728.679100:0:40798:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787828728.679103:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787828728.679104:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:3.0:1787828728.679106:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1787828728.679107:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1787828728.679108:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:0.0:1787828728.679109:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 00000020:00000001:3.0:1787828728.679112:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:3.0:1787828728.679115:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787828728.679116:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:0.0:1787828728.679118:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4298202880 00000100:00080000:3.0:1787828728.679119:0:40798:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:3.0:1787828728.679124:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828728.679125:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 02000000:00000001:3.0:1787828728.679126:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:0.0:1787828728.679130:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787828728.679131:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787828728.679136:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787828728.679136:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828728.679141:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00000001:3.0:1787828728.679142:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:3.0:1787828728.679146:0:40798:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1787828728.679149:0:40798:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828728.679151:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828728/real 1787828728] req@ffff8d30c70143c0 x1874670646758784/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828753 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787828728.679152:0:40798:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787828728.679158:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787828728.679161:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828728.679163:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000400:00000001:0.0:1787828728.679164:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 02000000:00000001:3.0:1787828728.679165:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000400:00000001:0.0:1787828728.679167:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 02000000:00000001:3.0:1787828728.679168:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828728.679172:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787828728.679174:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828728.679176:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787828728.679178:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787828728.679182:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c7014b40 x1874670646759168/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787828728.679183:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828728.679186:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787828728.679195:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c70143c0 x1874670646758784/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828753 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787828728.679195:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787828728.679199:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828728.679204:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828728.679208:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828728.679212:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787828728.679214:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828728.679216:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828728.679217:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828728.679218:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787828728.679220:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787828728.679222:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828728.679224:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787828728.679226:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787828728.679229:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c70143c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646758784:192.168.203.143@tcp:250:kworker.0 00000100:00000001:0.0:1787828728.679233:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828728.679235:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828728.679237:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787828728.679239:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787828728.679241:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787828728.679243:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787828728.679245:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787828728.679248:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787828728.679249:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787828728.679251:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787828728.679252:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828728.679253:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828728.679259:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787828728.679260:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828728.679262:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787828728.679263:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787828728.679264:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1787828728.679265:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787828728.679266:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828728.679268:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787828728.679269:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787828728.679271:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828728.679273:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c7015680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646758912:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787828728.679275:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787828728.679276:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787828728.679277:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828728.679279:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1787828728.679280:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828728.679284:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787828728.679285:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369541248 : -126234340010368 : ffff8d30c7015680) 00000100:00000001:0.0:1787828728.679289:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1787828728.679297:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:0.0:1787828728.679300:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1787828728.679301:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828728.679302:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828728.679303:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787828728.679305:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828728.679308:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298201622 00000100:00000001:0.0:1787828728.679310:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298202880 00000100:00000001:0.0:1787828728.679312:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298201622 00000100:00000001:0.0:1787828728.679313:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298202880 00000100:00000001:0.0:1787828728.679315:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787828728.679316:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828728.679317:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787828728.679320:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828728/real 1787828728] req@ffff8d30c7015680 x1874670646758912/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828753 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787828728.679325:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787828728.679326:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787828728.679327:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787828728.679328:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787828728.679329:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828728.679330:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828728.679331:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787828728.679333:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c7015680 x1874670646758912/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828753 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787828728.679338:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828728.679340:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787828728.679341:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828728.679342:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828728.679343:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828728.679344:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787828728.679345:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787828728.679347:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828728.679349:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787828728.679350:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787828728.679352:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c7015680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646758912:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787828728.679354:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1787828728.679355:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787828728.679355:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828728.679357:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787828728.679358:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787828728.679359:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828728.679361:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c70152c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646759040:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787828728.679374:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787828728.679375:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787828728.679376:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828728.679377:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1787828728.679379:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828728.679381:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787828728.679382:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369540288 : -126234340011328 : ffff8d30c70152c0) 00000100:00000001:0.0:1787828728.679384:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1787828728.679390:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:0.0:1787828728.679392:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1787828728.679393:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828728.679395:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828728.679396:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787828728.679397:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828728.679399:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298201622 00000100:00000001:0.0:1787828728.679401:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298202880 00000100:00000001:0.0:1787828728.679402:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298201622 00000100:00000001:0.0:1787828728.679403:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298202880 00000100:00000001:0.0:1787828728.679405:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787828728.679406:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828728.679407:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787828728.679409:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828728/real 1787828728] req@ffff8d30c70152c0 x1874670646759040/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828753 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787828728.679413:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787828728.679414:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787828728.679415:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787828728.679416:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787828728.679417:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828728.679418:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828728.679419:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787828728.679421:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c70152c0 x1874670646759040/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828753 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787828728.679425:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828728.679427:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787828728.679428:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828728.679438:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828728.679438:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828728.679439:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787828728.679440:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787828728.679442:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828728.679443:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787828728.679445:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787828728.679447:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c70152c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646759040:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787828728.679449:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1787828728.679450:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787828728.679451:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828728.679453:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787828728.679454:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787828728.679455:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828728.679457:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c7014b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646759168:192.168.203.143@tcp:8:kworker.0 00000100:00000001:0.0:1787828728.679458:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787828728.679459:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787828728.679460:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828728.679461:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1787828728.679463:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828728.679465:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787828728.679466:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369538368 : -126234340013248 : ffff8d30c7014b40) 00000100:00000001:0.0:1787828728.679468:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1787828728.679473:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:0.0:1787828728.679475:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1787828728.679476:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828728.679477:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828728.679478:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787828728.679479:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828728.679481:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298201635 00000100:00000001:0.0:1787828728.679483:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298202880 00000100:00000001:0.0:1787828728.679484:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298201635 00000100:00000001:0.0:1787828728.679486:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298202880 00000100:00000001:0.0:1787828728.679487:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787828728.679488:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828728.679489:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787828728.679499:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828728/real 1787828728] req@ffff8d30c7014b40 x1874670646759168/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828753 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787828728.679508:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787828728.679509:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787828728.679511:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787828728.679512:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787828728.679513:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828728.679514:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828728.679515:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787828728.679517:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c7014b40 x1874670646759168/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828753 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787828728.679522:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828728.679523:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787828728.679524:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828728.679525:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828728.679526:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828728.679527:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787828728.679528:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787828728.679529:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828728.679531:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787828728.679532:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787828728.679534:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c7014b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646759168:192.168.203.143@tcp:8:kworker.0 00000100:00000001:0.0:1787828728.679536:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828728.679537:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828728.679538:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787828728.679539:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787828728.679541:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787828728.679541:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787828728.679542:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787828728.679544:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787828728.679545:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787828728.679547:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787828728.679547:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828728.679549:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828728.679550:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787828728.679550:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787828728.679551:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787828728.679552:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787828728.679553:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787828728.679554:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787828728.679555:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787828728.679556:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787828728.679556:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828728.679557:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828728.679558:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787828728.679559:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787828728.679559:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787828728.679560:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787828728.679561:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787828728.679562:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787828728.679562:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787828728.679563:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787828728.679564:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828728.679565:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828728.679567:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787828728.679568:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828728.679569:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787828728.679570:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828729.574693:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828729.574699:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828729.574772:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828729.574776:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828729.574781:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828730.598686:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828730.598693:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828730.598757:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828730.598770:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828730.598775:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828731.622703:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828731.622721:0:815:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1787828731.622724:0:815:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1787828731.622727:0:815:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828731.622732:0:815:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1787828731.622736:0:815:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828731.622739:0:815:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828731.622743:0:815:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828731.622812:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828731.622817:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828731.622820:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828732.646744:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828732.646753:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828732.646825:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828732.646830:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828732.646833:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828733.670674:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828733.670679:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828733.670727:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828733.670731:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828733.670740:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787828733.798681:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828733.798690:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1787828733.798692:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787828733.798697:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787828733.798700:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787828733.798707:0:468:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 3534 00000100:00080000:2.0:1787828733.798716:0:468:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787828733.798718:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828733.798720:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787828733.798724:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787828733.798725:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787828733.798728:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:2.0:1787828733.798730:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:2.0:1787828733.798731:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828733.798732:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:2.0:1787828733.798735:0:468:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787828733.798737:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828733.798739:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787828733.798741:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828733.798743:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828733.798745:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:2.0:1787828733.798746:0:468:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828733.798767:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787828733.798768:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828733.798771:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787828733.798773:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787828733.798776:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828733.798786:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787828733.798793:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2f0c0 x1874670646759296/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787828733.798801:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787828733.798804:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828733.798817:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787828733.798821:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828733.798825:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1787828733.798827:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787828733.798828:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787828733.798831:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787828733.798834:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3534 00000100:00080000:2.0:1787828733.798838:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787828733.798840:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828733.798841:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787828733.798844:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787828733.798845:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:0.0:1787828733.798846:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:2.0:1787828733.798848:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000020:00000001:2.0:1787828733.798850:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:0.0:1787828733.798851:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1787828733.798852:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828733.798853:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:0.0:1787828733.798854:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1787828733.798856:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00080000:2.0:1787828733.798857:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787828733.798858:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828733.798859:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828733.798872:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787828733.798874:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787828733.798877:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828733.798877:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828733.798879:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828733.798880:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828733.798881:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00100000:0.0:1787828733.798884:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646759296:192.168.203.143@tcp:250:kworker.0 00000008:00100000:2.0:1787828733.798885:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00000001:0.0:1787828733.798889:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000008:00000001:2.0:1787828733.798889:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828733.798891:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828733.798893:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787828733.798895:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828733.798895:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787828733.798896:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828733.798898:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:2.0:1787828733.798898:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787828733.798900:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787828733.798901:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828733.798903:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828733.798906:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787828733.798911:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2e940 x1874670646759424/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787828733.798916:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787828733.798919:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417880768 : -126234291670848 : ffff8d30c9e2f0c0) 00000100:00000001:0.0:1787828733.798922:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:2.0:1787828733.798924:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787828733.798926:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828733.798929:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787828733.798932:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828733.798935:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1787828733.798936:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787828733.798938:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787828733.798940:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787828733.798943:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3534 00000100:00080000:2.0:1787828733.798946:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787828733.798947:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828733.798962:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787828733.798964:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787828733.798966:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787828733.798968:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000800:00000100:0.0:1787828733.798981:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000020:00000001:2.0:1787828733.798982:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:2.0:1787828733.798984:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828733.798985:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:2.0:1787828733.798988:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787828733.798991:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828733.798992:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:0.0:1787828733.798993:0:2390:0:(events.c:34:request_out_callback()) Process entered 02000000:00000001:2.0:1787828733.798995:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787828733.798997:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 02000000:00000001:2.0:1787828733.798997:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828733.798999:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828733.798999:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1787828733.799001:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000008:00100000:2.0:1787828733.799001:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1787828733.799004:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828733.799005:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828733.799007:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828733.799009:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787828733.799010:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828733.799011:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4298208000 00000100:00000001:2.0:1787828733.799012:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787828733.799014:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787828733.799015:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828733.799020:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 00000100:00000001:2.0:1787828733.799020:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787828733.799025:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2c780 x1874670646759552/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787828733.799026:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4298208000 00000100:00000001:0.0:1787828733.799030:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1787828733.799035:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787828733.799038:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828733.799039:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787828733.799041:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00000001:2.0:1787828733.799041:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828733.799043:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828733.799046:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828733/real 1787828733] req@ffff8d30c9e2f0c0 x1874670646759296/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828758 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828733.799046:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828733.799050:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1787828733.799051:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787828733.799053:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787828733.799055:0:468:0:(import.c:485:import_select_connection()) Process entered 00000400:00000001:0.0:1787828733.799057:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00080000:2.0:1787828733.799058:0:468:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3534 00000400:00000001:0.0:1787828733.799059:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00080000:2.0:1787828733.799061:0:468:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787828733.799063:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1787828733.799063:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828733.799064:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:0.0:1787828733.799065:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787828733.799067:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828733.799069:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000020:00000001:2.0:1787828733.799070:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:0.0:1787828733.799072:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:2.0:1787828733.799072:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787828733.799074:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00100000:0.0:1787828733.799076:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2f0c0 x1874670646759296/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828758 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:2.0:1787828733.799076:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:2.0:1787828733.799078:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828733.799080:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:2.0:1787828733.799083:0:468:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787828733.799084:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828733.799086:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828733.799087:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:0.0:1787828733.799088:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 02000000:00000001:2.0:1787828733.799089:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787828733.799091:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:2.0:1787828733.799091:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828733.799093:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828733.799093:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1787828733.799094:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000008:00100000:2.0:1787828733.799095:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00000001:0.0:1787828733.799097:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000008:00000001:2.0:1787828733.799098:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828733.799099:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787828733.799100:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787828733.799102:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 02000000:00000001:2.0:1787828733.799104:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787828733.799105:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787828733.799107:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:2.0:1787828733.799107:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:0.0:1787828733.799109:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 02000000:00000001:2.0:1787828733.799109:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787828733.799110:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828733.799113:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646759296:192.168.203.143@tcp:250:kworker.0 00000100:00000001:2.0:1787828733.799114:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828733.799118:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787828733.799118:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2c3c0 x1874670646759680/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787828733.799120:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828733.799122:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787828733.799124:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 00000100:00000001:2.0:1787828733.799126:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 02000000:00000001:0.0:1787828733.799127:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00000001:2.0:1787828733.799128:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787828733.799129:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000100:00000001:2.0:1787828733.799130:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787828733.799132:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787828733.799135:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787828733.799137:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787828733.799141:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787828733.799143:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828733.799145:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828733.799161:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787828733.799164:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828733.799166:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787828733.799168:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787828733.799170:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1787828733.799171:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787828733.799173:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828733.799175:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787828733.799176:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787828733.799178:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828733.799181:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646759424:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787828733.799183:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787828733.799185:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787828733.799186:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828733.799187:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1787828733.799189:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828733.799194:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787828733.799195:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417878848 : -126234291672768 : ffff8d30c9e2e940) 00000100:00000001:0.0:1787828733.799199:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1787828733.799209:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:0.0:1787828733.799212:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1787828733.799214:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828733.799215:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828733.799216:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787828733.799218:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828733.799221:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298206742 00000100:00000001:0.0:1787828733.799223:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298208000 00000100:00000001:0.0:1787828733.799225:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298206742 00000100:00000001:0.0:1787828733.799227:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298208000 00000100:00000001:0.0:1787828733.799228:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787828733.799230:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828733.799231:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787828733.799234:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828733/real 1787828733] req@ffff8d30c9e2e940 x1874670646759424/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828758 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787828733.799239:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787828733.799239:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787828733.799241:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787828733.799242:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787828733.799243:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828733.799244:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828733.799245:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787828733.799247:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2e940 x1874670646759424/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828758 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787828733.799252:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828733.799253:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787828733.799255:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828733.799256:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828733.799256:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828733.799258:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787828733.799259:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787828733.799260:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828733.799262:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787828733.799264:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787828733.799266:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646759424:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787828733.799269:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1787828733.799270:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787828733.799270:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828733.799272:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787828733.799273:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787828733.799274:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828733.799275:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646759552:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787828733.799277:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787828733.799278:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787828733.799279:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828733.799280:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1787828733.799282:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828733.799285:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787828733.799286:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417870208 : -126234291681408 : ffff8d30c9e2c780) 00000100:00000001:0.0:1787828733.799288:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1787828733.799294:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:0.0:1787828733.799296:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1787828733.799298:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828733.799299:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828733.799300:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787828733.799301:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828733.799303:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298206742 00000100:00000001:0.0:1787828733.799305:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298208000 00000100:00000001:0.0:1787828733.799306:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298206742 00000100:00000001:0.0:1787828733.799308:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298208000 00000100:00000001:0.0:1787828733.799309:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787828733.799311:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828733.799312:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787828733.799314:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828733/real 1787828733] req@ffff8d30c9e2c780 x1874670646759552/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828758 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787828733.799319:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787828733.799320:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787828733.799321:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787828733.799322:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787828733.799323:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828733.799324:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828733.799325:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787828733.799327:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2c780 x1874670646759552/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828758 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787828733.799332:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828733.799334:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787828733.799335:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828733.799336:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828733.799337:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828733.799338:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787828733.799338:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787828733.799340:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828733.799342:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787828733.799343:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787828733.799345:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646759552:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787828733.799346:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1787828733.799347:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787828733.799348:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828733.799356:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787828733.799357:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787828733.799359:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828733.799361:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646759680:192.168.203.143@tcp:8:kworker.0 00000100:00000001:0.0:1787828733.799362:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787828733.799364:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787828733.799365:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828733.799366:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1787828733.799368:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828733.799371:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787828733.799372:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417869248 : -126234291682368 : ffff8d30c9e2c3c0) 00000100:00000001:0.0:1787828733.799374:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1787828733.799381:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:0.0:1787828733.799383:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1787828733.799384:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828733.799385:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828733.799387:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787828733.799388:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828733.799390:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298206755 00000100:00000001:0.0:1787828733.799392:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298208000 00000100:00000001:0.0:1787828733.799393:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298206755 00000100:00000001:0.0:1787828733.799395:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298208000 00000100:00000001:0.0:1787828733.799396:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787828733.799397:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828733.799398:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787828733.799401:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828733/real 1787828733] req@ffff8d30c9e2c3c0 x1874670646759680/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828758 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787828733.799406:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787828733.799407:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787828733.799408:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787828733.799409:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787828733.799410:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828733.799411:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828733.799412:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787828733.799414:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2c3c0 x1874670646759680/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828758 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787828733.799418:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828733.799420:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787828733.799422:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828733.799423:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828733.799423:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828733.799424:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787828733.799426:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787828733.799427:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828733.799429:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787828733.799430:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787828733.799432:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646759680:192.168.203.143@tcp:8:kworker.0 00000100:00000001:0.0:1787828733.799434:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828733.799435:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828733.799436:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787828733.799437:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787828733.799438:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787828733.799439:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787828733.799440:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787828733.799441:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787828733.799442:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787828733.799445:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787828733.799446:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828733.799447:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828733.799447:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787828733.799448:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787828733.799449:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787828733.799450:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787828733.799451:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787828733.799452:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787828733.799453:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787828733.799454:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787828733.799455:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828733.799456:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828733.799457:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787828733.799458:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787828733.799458:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787828733.799459:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787828733.799460:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787828733.799461:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787828733.799462:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787828733.799463:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787828733.799464:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828733.799465:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828733.799467:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787828733.799469:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828733.799470:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787828733.799471:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828734.694660:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828734.694665:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828734.694712:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828734.694716:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828734.694718:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828735.719644:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828735.719649:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828735.719721:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828735.719725:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828735.719728:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828736.742670:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828736.742675:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828736.742719:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828736.742723:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828736.742725:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828737.766669:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828737.766675:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828737.766718:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828737.766721:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828737.766724:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828738.790660:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828738.790665:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828738.790752:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828738.790756:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828738.790758:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787828738.918695:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828738.918702:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1787828738.918704:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787828738.918707:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787828738.918711:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787828738.918717:0:468:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 3539 00000100:00080000:0.0:1787828738.918726:0:468:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787828738.918729:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787828738.918730:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787828738.918734:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787828738.918736:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787828738.918738:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:0.0:1787828738.918740:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:0.0:1787828738.918742:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787828738.918743:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:0.0:1787828738.918746:0:468:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787828738.918748:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828738.918750:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787828738.918752:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787828738.918755:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828738.918756:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1787828738.918758:0:468:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828738.918768:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787828738.918769:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828738.918771:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787828738.918774:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787828738.918777:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828738.918788:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828738.918795:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d26940 x1874670646759808/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787828738.918803:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787828738.918807:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828738.918821:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787828738.918824:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828738.918826:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1787828738.918827:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787828738.918828:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787828738.918830:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787828738.918832:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3539 00000100:00080000:0.0:1787828738.918835:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787828738.918837:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787828738.918838:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787828738.918840:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787828738.918841:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787828738.918843:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000020:00000001:0.0:1787828738.918845:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:0.0:1787828738.918846:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828738.918846:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787828738.918851:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:2.0:1787828738.918853:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00080000:0.0:1787828738.918855:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787828738.918856:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1787828738.918858:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828738.918859:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 02000000:00000001:0.0:1787828738.918861:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:2.0:1787828738.918862:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828738.918866:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828738.918867:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787828738.918871:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828738.918872:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787828738.918875:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 02000000:00000001:2.0:1787828738.918877:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000008:00100000:0.0:1787828738.918880:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828738.918882:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d26940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646759808:192.168.203.143@tcp:250:kworker.0 00000008:00000001:0.0:1787828738.918884:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828738.918888:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828738.918890:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787828738.918892:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787828738.918895:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828738.918895:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1787828738.918899:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828738.918900:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787828738.918902:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787828738.918904:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1787828738.918905:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828738.918910:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828738.918912:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828738.918932:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d25680 x1874670646759936/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787828738.918936:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787828738.918939:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601346880 : -126234108204736 : ffff8d30d4d26940) 00000100:00000001:0.0:1787828738.918942:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787828738.918945:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828738.918945:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:0.0:1787828738.918951:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787828738.918953:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828738.918955:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1787828738.918956:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787828738.918958:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787828738.918960:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787828738.918962:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3539 00000100:00080000:0.0:1787828738.918970:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787828738.918972:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787828738.918973:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787828738.918975:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787828738.918977:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000800:00000100:2.0:1787828738.918978:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000020:00000001:0.0:1787828738.918979:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000020:00000001:0.0:1787828738.918983:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:0.0:1787828738.918985:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787828738.918986:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:0.0:1787828738.918990:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787828738.918991:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1787828738.918992:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828738.918995:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:2.0:1787828738.918996:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 02000000:00000001:0.0:1787828738.918998:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787828738.919000:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828738.919002:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828738.919003:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787828738.919007:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1787828738.919010:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00000001:2.0:1787828738.919010:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:0.0:1787828738.919014:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828738.919016:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4298213120 00000100:00000001:0.0:1787828738.919017:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828738.919024:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787828738.919025:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828738.919026:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787828738.919027:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:2.0:1787828738.919027:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 02000000:00000001:0.0:1787828738.919031:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828738.919036:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828738.919037:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4298213120 00000100:00100000:0.0:1787828738.919043:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d252c0 x1874670646760064/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787828738.919044:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 00000100:00000001:2.0:1787828738.919049:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787828738.919054:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828738.919058:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00000001:0.0:1787828738.919062:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00100000:2.0:1787828738.919065:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828738/real 1787828738] req@ffff8d30d4d26940 x1874670646759808/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828763 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787828738.919067:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828738.919074:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787828738.919082:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000001:2.0:1787828738.919082:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787828738.919084:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1787828738.919088:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00080000:0.0:1787828738.919089:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1787828738.919091:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787828738.919094:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:2.0:1787828738.919094:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:0.0:1787828738.919100:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787828738.919100:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828738.919103:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828738.919107:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00100000:2.0:1787828738.919109:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d26940 x1874670646759808/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828763 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828738.919112:0:468:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3539 00000100:00080000:0.0:1787828738.919120:0:468:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787828738.919121:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828738.919123:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828738.919125:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787828738.919126:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787828738.919132:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:2.0:1787828738.919132:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1787828738.919135:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:2.0:1787828738.919136:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000020:00000001:0.0:1787828738.919138:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:2.0:1787828738.919139:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1787828738.919144:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:2.0:1787828738.919145:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787828738.919149:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828738.919149:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787828738.919152:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:2.0:1787828738.919154:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828738.919157:0:468:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00080000:2.0:1787828738.919161:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787828738.919163:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828738.919164:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 02000000:00000001:0.0:1787828738.919166:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787828738.919170:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00100000:2.0:1787828738.919170:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d26940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646759808:192.168.203.143@tcp:250:kworker.0 02000000:00000001:0.0:1787828738.919176:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828738.919177:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828738.919179:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:2.0:1787828738.919180:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000008:00100000:0.0:1787828738.919182:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00000001:2.0:1787828738.919183:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 00000008:00000001:0.0:1787828738.919185:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828738.919186:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 00000100:00000001:0.0:1787828738.919188:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828738.919190:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 02000000:00000001:0.0:1787828738.919193:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000020:00000001:2.0:1787828738.919193:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:2.0:1787828738.919194:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787828738.919196:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828738.919199:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787828738.919199:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787828738.919203:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787828738.919203:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 02000000:00000001:0.0:1787828738.919205:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828738.919207:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787828738.919210:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828738.919210:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828738.919212:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787828738.919218:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d27c00 x1874670646760192/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787828738.919220:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1787828738.919223:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828738.919228:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787828738.919229:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787828738.919230:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787828738.919232:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828738.919233:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1787828738.919236:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787828738.919238:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828738.919238:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828738.919244:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787828738.919245:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828738.919247:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787828738.919250:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d25680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646759936:192.168.203.143@tcp:38:kworker.0 00000100:00000001:2.0:1787828738.919252:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787828738.919254:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1787828738.919255:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828738.919256:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1787828738.919259:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828738.919263:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787828738.919266:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601342080 : -126234108209536 : ffff8d30d4d25680) 00000100:00000001:2.0:1787828738.919272:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1787828738.919285:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:2.0:1787828738.919290:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1787828738.919293:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828738.919294:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828738.919296:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1787828738.919299:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828738.919304:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298211862 00000100:00000001:2.0:1787828738.919307:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298213120 00000100:00000001:2.0:1787828738.919310:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298211862 00000100:00000001:2.0:1787828738.919313:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298213120 00000100:00000001:2.0:1787828738.919315:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787828738.919317:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828738.919319:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1787828738.919323:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828738/real 1787828738] req@ffff8d30d4d25680 x1874670646759936/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828763 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1787828738.919329:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787828738.919330:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1787828738.919333:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1787828738.919334:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787828738.919335:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828738.919336:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828738.919337:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787828738.919340:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d25680 x1874670646759936/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828763 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787828738.919345:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828738.919346:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787828738.919348:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828738.919349:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828738.919350:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828738.919351:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787828738.919368:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787828738.919371:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828738.919373:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:2.0:1787828738.919375:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1787828738.919378:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d25680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646759936:192.168.203.143@tcp:38:kworker.0 00000100:00000001:2.0:1787828738.919381:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1787828738.919382:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787828738.919383:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828738.919385:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787828738.919386:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828738.919388:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787828738.919390:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d252c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646760064:192.168.203.143@tcp:38:kworker.0 00000100:00000001:2.0:1787828738.919393:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787828738.919395:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1787828738.919396:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828738.919398:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1787828738.919401:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828738.919405:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787828738.919406:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601341120 : -126234108210496 : ffff8d30d4d252c0) 00000100:00000001:2.0:1787828738.919410:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1787828738.919437:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:2.0:1787828738.919441:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1787828738.919443:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828738.919444:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828738.919446:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1787828738.919448:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828738.919451:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298211862 00000100:00000001:2.0:1787828738.919454:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298213120 00000100:00000001:2.0:1787828738.919456:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298211862 00000100:00000001:2.0:1787828738.919458:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298213120 00000100:00000001:2.0:1787828738.919461:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787828738.919463:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828738.919465:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1787828738.919469:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828738/real 1787828738] req@ffff8d30d4d252c0 x1874670646760064/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828763 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1787828738.919479:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787828738.919480:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1787828738.919483:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1787828738.919484:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787828738.919486:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828738.919488:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828738.919489:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787828738.919493:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d252c0 x1874670646760064/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828763 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787828738.919501:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828738.919503:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787828738.919505:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828738.919507:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828738.919508:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828738.919510:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787828738.919512:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787828738.919515:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828738.919534:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:2.0:1787828738.919537:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1787828738.919541:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d252c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646760064:192.168.203.143@tcp:38:kworker.0 00000100:00000001:2.0:1787828738.919546:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828738.919549:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828738.919550:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787828738.919552:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1787828738.919555:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787828738.919557:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:2.0:1787828738.919558:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:2.0:1787828738.919560:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787828738.919561:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1787828738.919564:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787828738.919565:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828738.919566:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828738.919567:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787828738.919568:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1787828738.919570:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787828738.919571:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:2.0:1787828738.919572:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:2.0:1787828738.919573:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787828738.919574:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1787828738.919575:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787828738.919575:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828738.919577:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828738.919580:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1787828738.919581:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828738.919582:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787828738.919583:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1787828738.919584:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1787828738.919585:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787828738.919586:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828738.919588:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787828738.919590:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828738.919591:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787828738.919594:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d27c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646760192:192.168.203.143@tcp:8:kworker.0 00000100:00000001:2.0:1787828738.919597:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787828738.919598:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1787828738.919599:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828738.919601:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1787828738.919603:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828738.919610:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787828738.919611:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601351680 : -126234108199936 : ffff8d30d4d27c00) 00000100:00000001:2.0:1787828738.919615:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1787828738.919643:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:2.0:1787828738.919648:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1787828738.919650:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828738.919652:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828738.919654:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1787828738.919656:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828738.919680:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298211876 00000100:00000001:2.0:1787828738.919700:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298213120 00000100:00000001:2.0:1787828738.919702:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298211876 00000100:00000001:2.0:1787828738.919704:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298213120 00000100:00000001:2.0:1787828738.919705:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787828738.919707:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828738.919708:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1787828738.919712:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828738/real 1787828738] req@ffff8d30d4d27c00 x1874670646760192/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828763 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1787828738.919718:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787828738.919719:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1787828738.919722:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1787828738.919723:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787828738.919724:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828738.919726:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828738.919727:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787828738.919729:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d27c00 x1874670646760192/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828763 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787828738.919735:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828738.919736:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787828738.919738:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828738.919739:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828738.919740:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828738.919742:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787828738.919743:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787828738.919745:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828738.919747:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:2.0:1787828738.919748:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1787828738.919750:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d27c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646760192:192.168.203.143@tcp:8:kworker.0 00000100:00000001:2.0:1787828738.919755:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828738.919757:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828738.919758:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787828738.919759:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1787828738.919761:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787828738.919762:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:2.0:1787828738.919763:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:2.0:1787828738.919765:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787828738.919766:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1787828738.919768:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787828738.919769:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828738.919770:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828738.919773:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1787828738.919774:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828738.919776:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787828738.919777:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828739.814699:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828739.814715:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828739.814803:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828739.814807:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828739.814811:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828740.838710:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828740.838716:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828740.838779:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828740.838782:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828740.838785:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828741.862701:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828741.862708:0:815:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1787828741.862710:0:815:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1787828741.862712:0:815:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828741.862716:0:815:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1787828741.862717:0:815:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828741.862719:0:815:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828741.862721:0:815:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828741.862778:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828741.862783:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828741.862787:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828742.886821:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828742.886831:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828742.886923:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828742.886929:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828742.886934:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828743.910795:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828743.910809:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828743.910928:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828743.910934:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828743.910938:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787828744.038786:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828744.038814:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1787828744.038817:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787828744.038822:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787828744.038829:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787828744.038839:0:40798:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 3544 00000100:00080000:0.0:1787828744.038852:0:40798:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787828744.038856:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787828744.038859:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787828744.038865:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787828744.038868:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787828744.038872:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:0.0:1787828744.038875:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:0.0:1787828744.038908:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787828744.038910:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:0.0:1787828744.038917:0:40798:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787828744.038932:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828744.038936:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787828744.038940:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787828744.038944:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828744.038947:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1787828744.038951:0:40798:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828744.038966:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787828744.038968:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828744.038972:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787828744.038975:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787828744.038981:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828744.038997:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828744.039021:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d270c0 x1874670646760320/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787828744.039036:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787828744.039043:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828744.039082:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787828744.039087:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828744.039092:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1787828744.039095:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787828744.039097:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787828744.039102:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787828744.039106:0:40798:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3544 00000100:00080000:0.0:1787828744.039113:0:40798:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787828744.039116:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787828744.039118:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787828744.039122:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787828744.039124:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787828744.039128:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000020:00000001:0.0:1787828744.039132:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:0.0:1787828744.039147:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787828744.039149:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:0.0:1787828744.039152:0:40798:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787828744.039155:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828744.039156:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787828744.039162:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787828744.039164:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828744.039166:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1787828744.039171:0:40798:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1787828744.039174:0:40798:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828744.039176:0:40798:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828744.039180:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787828744.039182:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828744.039184:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787828744.039186:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787828744.039188:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828744.039194:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828744.039200:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d24000 x1874670646760448/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787828744.039220:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787828744.039222:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828744.039225:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787828744.039229:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828744.039233:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1787828744.039235:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787828744.039237:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787828744.039240:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787828744.039242:0:40798:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3544 00000100:00080000:0.0:1787828744.039245:0:40798:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787828744.039247:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787828744.039249:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787828744.039256:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:2.0:1787828744.039257:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:0.0:1787828744.039259:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787828744.039267:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000020:00000001:0.0:1787828744.039273:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:2.0:1787828744.039275:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787828744.039285:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828744.039286:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1787828744.039293:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:2.0:1787828744.039295:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787828744.039302:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787828744.039304:0:40798:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787828744.039328:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828744.039329:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787828744.039337:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787828744.039338:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787828744.039357:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828744.039365:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828744.039367:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828744.039376:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00100000:2.0:1787828744.039383:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d270c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646760320:192.168.203.143@tcp:250:kworker.0 00000008:00100000:0.0:1787828744.039384:0:40798:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1787828744.039403:0:40798:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828744.039404:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:0.0:1787828744.039412:0:40798:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828744.039420:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787828744.039421:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787828744.039422:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828744.039428:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787828744.039430:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787828744.039431:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828744.039434:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828744.039446:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:0.0:1787828744.039447:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828744.039460:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d24f00 x1874670646760576/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787828744.039468:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 02000000:00000001:2.0:1787828744.039470:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828744.039471:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828744.039486:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787828744.039491:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828744.039494:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1787828744.039495:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787828744.039498:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787828744.039500:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787828744.039515:0:40798:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3544 00000100:00080000:0.0:1787828744.039518:0:40798:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787828744.039520:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787828744.039522:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787828744.039524:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787828744.039525:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787828744.039527:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000020:00000001:0.0:1787828744.039528:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:0.0:1787828744.039531:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828744.039532:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787828744.039534:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:2.0:1787828744.039539:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601348800 : -126234108202816 : ffff8d30d4d270c0) 00000100:00080000:0.0:1787828744.039540:0:40798:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787828744.039547:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828744.039555:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:2.0:1787828744.039559:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 02000000:00000001:0.0:1787828744.039560:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787828744.039565:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828744.039567:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1787828744.039570:0:40798:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1787828744.039572:0:40798:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828744.039573:0:40798:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828744.039577:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787828744.039578:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828744.039580:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787828744.039581:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787828744.039583:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828744.039588:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828744.039596:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d25a40 x1874670646760704/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787828744.039602:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787828744.039603:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828744.039605:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:2.0:1787828744.039708:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:2.0:1787828744.039732:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1787828744.039757:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828744.039761:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828744.039764:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1787828744.039774:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828744.039781:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4298218241 00000100:00000001:2.0:1787828744.039797:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 00000100:00000001:2.0:1787828744.039835:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4298218241 00000100:00000001:2.0:1787828744.039842:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 00000100:00000001:2.0:1787828744.039847:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787828744.039851:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828744.039857:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1787828744.039872:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828744/real 1787828744] req@ffff8d30d4d270c0 x1874670646760320/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828769 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1787828744.039918:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787828744.039921:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1787828744.039927:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1787828744.039931:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787828744.039933:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828744.039937:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828744.039940:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787828744.039948:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d270c0 x1874670646760320/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828769 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787828744.039968:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828744.039974:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787828744.039978:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828744.039980:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828744.039982:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828744.039986:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787828744.039989:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787828744.039993:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828744.039999:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:2.0:1787828744.040002:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1787828744.040023:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d270c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646760320:192.168.203.143@tcp:250:kworker.0 00000100:00000001:2.0:1787828744.040030:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1787828744.040033:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787828744.040035:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828744.040038:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787828744.040041:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828744.040044:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787828744.040048:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d24000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646760448:192.168.203.143@tcp:38:kworker.0 00000100:00000001:2.0:1787828744.040052:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787828744.040064:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1787828744.040067:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828744.040069:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1787828744.040073:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828744.040082:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787828744.040083:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601336320 : -126234108215296 : ffff8d30d4d24000) 00000100:00000001:2.0:1787828744.040090:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1787828744.040113:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:2.0:1787828744.040122:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1787828744.040126:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828744.040129:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828744.040148:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1787828744.040155:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828744.040165:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298216983 00000100:00000001:2.0:1787828744.040171:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298218241 00000100:00000001:2.0:1787828744.040176:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298216983 00000100:00000001:2.0:1787828744.040181:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298218241 00000100:00000001:2.0:1787828744.040185:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787828744.040190:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828744.040194:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1787828744.040219:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828744/real 1787828744] req@ffff8d30d4d24000 x1874670646760448/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828769 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1787828744.040235:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787828744.040238:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1787828744.040243:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1787828744.040245:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787828744.040247:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828744.040249:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828744.040251:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787828744.040257:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d24000 x1874670646760448/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828769 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787828744.040265:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828744.040268:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787828744.040271:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828744.040273:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828744.040275:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828744.040277:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787828744.040279:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787828744.040282:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828744.040286:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:2.0:1787828744.040288:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1787828744.040292:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d24000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646760448:192.168.203.143@tcp:38:kworker.0 00000100:00000001:2.0:1787828744.040298:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828744.040301:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828744.040303:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787828744.040318:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1787828744.040323:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787828744.040327:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:2.0:1787828744.040329:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:2.0:1787828744.040336:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787828744.040338:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1787828744.040361:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787828744.040363:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828744.040365:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828744.040367:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787828744.040369:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1787828744.040382:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787828744.040384:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:2.0:1787828744.040385:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:2.0:1787828744.040387:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787828744.040388:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1787828744.040392:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787828744.040393:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828744.040396:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828744.040408:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1787828744.040412:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828744.040417:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787828744.040420:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1787828744.040422:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1787828744.040426:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787828744.040429:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828744.040434:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787828744.040437:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828744.040442:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787828744.040449:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d24f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646760576:192.168.203.143@tcp:38:kworker.0 00000100:00000001:2.0:1787828744.040457:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787828744.040462:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1787828744.040465:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828744.040469:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1787828744.040475:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828744.040488:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787828744.040491:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601340160 : -126234108211456 : ffff8d30d4d24f00) 00000100:00000001:2.0:1787828744.040498:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1787828744.040542:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:2.0:1787828744.040551:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1787828744.040555:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828744.040558:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828744.040562:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1787828744.040566:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828744.040573:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298216983 00000100:00000001:2.0:1787828744.040578:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298218241 00000100:00000001:2.0:1787828744.040582:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298216983 00000100:00000001:2.0:1787828744.040587:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298218241 00000100:00000001:2.0:1787828744.040591:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787828744.040595:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828744.040599:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1787828744.040607:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828744/real 1787828744] req@ffff8d30d4d24f00 x1874670646760576/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828769 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1787828744.040624:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787828744.040634:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1787828744.040639:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1787828744.040642:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787828744.040644:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828744.040648:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828744.040650:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787828744.040658:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d24f00 x1874670646760576/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828769 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787828744.040672:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828744.040677:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787828744.040734:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828744.040737:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828744.040739:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828744.040742:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787828744.040761:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787828744.040765:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828744.040770:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:2.0:1787828744.040773:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1787828744.040778:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d24f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646760576:192.168.203.143@tcp:38:kworker.0 00000100:00000001:2.0:1787828744.040783:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1787828744.040786:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787828744.040788:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828744.040791:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787828744.040794:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828744.040805:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787828744.040809:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d25a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646760704:192.168.203.143@tcp:8:kworker.0 00000100:00000001:2.0:1787828744.040814:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787828744.040817:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1787828744.040820:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828744.040822:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1787828744.040826:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828744.040836:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787828744.040838:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601343040 : -126234108208576 : ffff8d30d4d25a40) 00000100:00000001:2.0:1787828744.040843:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1787828744.040861:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:2.0:1787828744.040868:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1787828744.040871:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828744.040873:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828744.040875:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1787828744.040909:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828744.040916:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298216997 00000100:00000001:2.0:1787828744.040937:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298218241 00000100:00000001:2.0:1787828744.040949:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298216997 00000100:00000001:2.0:1787828744.040952:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298218241 00000100:00000001:2.0:1787828744.040957:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787828744.040959:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828744.040962:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1787828744.040968:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828744/real 1787828744] req@ffff8d30d4d25a40 x1874670646760704/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828769 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1787828744.040978:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787828744.040981:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1787828744.040984:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1787828744.040986:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787828744.040988:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828744.040990:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828744.040991:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787828744.041020:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d25a40 x1874670646760704/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828769 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787828744.041034:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828744.041037:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787828744.041040:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828744.041042:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828744.041044:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828744.041046:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787828744.041048:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787828744.041051:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828744.041055:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:2.0:1787828744.041057:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1787828744.041061:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d25a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646760704:192.168.203.143@tcp:8:kworker.0 00000100:00000001:2.0:1787828744.041066:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828744.041069:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828744.041080:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787828744.041081:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1787828744.041084:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787828744.041086:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:2.0:1787828744.041088:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:2.0:1787828744.041091:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787828744.041093:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1787828744.041098:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787828744.041099:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828744.041101:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828744.041103:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787828744.041104:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1787828744.041106:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787828744.041108:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:2.0:1787828744.041109:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:2.0:1787828744.041111:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787828744.041112:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1787828744.041114:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787828744.041117:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828744.041120:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828744.041126:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1787828744.041128:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828744.041132:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787828744.041135:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828744.934721:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828744.934729:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828744.934792:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828744.934797:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828744.934800:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828745.958739:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828745.958747:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828745.958832:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828745.958837:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828745.958841:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828746.982683:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828746.982689:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828746.982747:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828746.982750:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828746.982753:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828748.006681:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828748.006686:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828748.006738:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828748.006741:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828748.006744:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828749.030682:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828749.030687:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828749.030744:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828749.030760:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828749.030763:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787828749.158824:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828749.158836:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1787828749.158840:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787828749.158896:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787828749.158902:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787828749.158910:0:468:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 3549 00000100:00080000:2.0:1787828749.158921:0:468:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787828749.158924:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828749.158927:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787828749.158932:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787828749.158935:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787828749.158939:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:2.0:1787828749.158943:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:2.0:1787828749.158946:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828749.158948:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:2.0:1787828749.158955:0:468:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787828749.158959:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828749.158963:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787828749.158967:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828749.158971:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828749.158975:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:2.0:1787828749.158978:0:468:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828749.159000:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787828749.159003:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828749.159007:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787828749.159011:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787828749.159015:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828749.159029:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787828749.159040:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2de00 x1874670646760832/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787828749.159054:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787828749.159059:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828749.159079:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787828749.159083:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828749.159088:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1787828749.159090:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787828749.159093:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787828749.159097:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787828749.159103:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3549 00000100:00080000:2.0:1787828749.159109:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787828749.159124:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787828749.159124:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828749.159129:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:0.0:1787828749.159133:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000020:00000001:2.0:1787828749.159135:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:0.0:1787828749.159140:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000020:00000001:2.0:1787828749.159141:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:0.0:1787828749.159148:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000020:00000001:2.0:1787828749.159148:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:0.0:1787828749.159154:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1787828749.159156:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:2.0:1787828749.159163:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 02000000:00000001:0.0:1787828749.159164:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:2.0:1787828749.159166:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 02000000:00000001:0.0:1787828749.159172:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00080000:2.0:1787828749.159174:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 02000000:00000001:0.0:1787828749.159178:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828749.159183:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828749.159198:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00100000:0.0:1787828749.159199:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646760832:192.168.203.143@tcp:250:kworker.0 02000000:00000001:2.0:1787828749.159206:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787828749.159207:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787828749.159214:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1787828749.159214:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828749.159219:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828749.159221:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 02000000:00000001:0.0:1787828749.159223:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000008:00100000:2.0:1787828749.159229:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 02000000:00000001:0.0:1787828749.159231:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:2.0:1787828749.159239:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828749.159244:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828749.159251:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:0.0:1787828749.159254:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 02000000:00000001:2.0:1787828749.159254:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828749.159260:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417875968 : -126234291675648 : ffff8d30c9e2de00) 00000100:00000001:2.0:1787828749.159261:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787828749.159265:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:0.0:1787828749.159269:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 02000000:00000001:2.0:1787828749.159271:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828749.159290:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787828749.159310:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2fc00 x1874670646760960/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787828749.159321:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787828749.159324:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828749.159329:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787828749.159333:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:0.0:1787828749.159336:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00080000:2.0:1787828749.159342:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1787828749.159347:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787828749.159350:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787828749.159354:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1787828749.159356:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:0.0:1787828749.159362:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00080000:2.0:1787828749.159366:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3549 00000100:00000001:0.0:1787828749.159367:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828749.159371:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00080000:2.0:1787828749.159378:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787828749.159380:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828749.159387:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787828749.159389:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4298223360 00000100:00000001:2.0:1787828749.159391:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787828749.159396:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787828749.159405:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787828749.159407:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000020:00000001:2.0:1787828749.159410:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:0.0:1787828749.159412:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 00000100:00000001:2.0:1787828749.159412:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828749.159418:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:0.0:1787828749.159422:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4298223360 00000100:00080000:2.0:1787828749.159424:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787828749.159430:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 00000100:00000001:2.0:1787828749.159431:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828749.159437:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828749.159438:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:0.0:1787828749.159443:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828749.159444:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787828749.159448:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 02000000:00000001:2.0:1787828749.159450:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828749.159454:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00100000:0.0:1787828749.159458:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828749/real 1787828749] req@ffff8d30c9e2de00 x1874670646760832/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828774 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:2.0:1787828749.159458:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000400:00000001:0.0:1787828749.159475:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000008:00000001:2.0:1787828749.159476:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:0.0:1787828749.159478:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1787828749.159481:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828749.159484:0:2390:0:(events.c:78:reply_in_callback()) Process entered 02000000:00000001:2.0:1787828749.159489:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:0.0:1787828749.159490:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 02000000:00000001:2.0:1787828749.159495:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828749.159496:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828749.159502:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:0.0:1787828749.159503:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 02000000:00000001:2.0:1787828749.159507:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:0.0:1787828749.159508:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:2.0:1787828749.159512:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828749.159516:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2de00 x1874670646760832/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828774 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787828749.159520:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828749.159530:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787828749.159532:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2d2c0 x1874670646761088/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787828749.159534:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787828749.159541:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828749.159546:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828749.159549:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828749.159554:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787828749.159555:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787828749.159560:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787828749.159561:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787828749.159566:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00000001:2.0:1787828749.159568:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787828749.159573:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828749.159574:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787828749.159580:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787828749.159584:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646760832:192.168.203.143@tcp:250:kworker.0 00000100:00080000:2.0:1787828749.159588:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1787828749.159591:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828749.159592:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:0.0:1787828749.159597:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00080000:2.0:1787828749.159597:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787828749.159626:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787828749.159629:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787828749.159633:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787828749.159635:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787828749.159636:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 00000100:00000001:2.0:1787828749.159642:0:468:0:(import.c:485:import_select_connection()) Process entered 02000000:00000001:0.0:1787828749.159643:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787828749.159648:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00080000:2.0:1787828749.159649:0:468:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3549 00000100:00000001:0.0:1787828749.159658:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00080000:2.0:1787828749.159658:0:468:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787828749.159662:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828749.159664:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787828749.159667:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828749.159668:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787828749.159674:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787828749.159675:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787828749.159677:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:0.0:1787828749.159678:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000020:00000001:2.0:1787828749.159680:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:0.0:1787828749.159682:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828749.159686:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787828749.159687:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787828749.159690:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:0.0:1787828749.159692:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787828749.159696:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00080000:2.0:1787828749.159699:0:468:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787828749.159700:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787828749.159703:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828749.159707:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828749.159708:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787828749.159714:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828749.159714:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787828749.159720:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828749.159720:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00100000:0.0:1787828749.159726:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646760960:192.168.203.143@tcp:38:kworker.0 02000000:00000001:2.0:1787828749.159727:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828749.159734:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1787828749.159735:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000008:00100000:2.0:1787828749.159738:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 02000000:00000001:0.0:1787828749.159739:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787828749.159744:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:2.0:1787828749.159745:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828749.159749:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:2.0:1787828749.159750:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828749.159753:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828749.159770:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:0.0:1787828749.159773:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 02000000:00000001:2.0:1787828749.159774:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828749.159777:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417883648 : -126234291667968 : ffff8d30c9e2fc00) 00000100:00000001:2.0:1787828749.159779:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787828749.159783:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:0.0:1787828749.159786:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 02000000:00000001:2.0:1787828749.159787:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828749.159796:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787828749.159803:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2cb40 x1874670646761216/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787828749.159807:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:0.0:1787828749.159815:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1787828749.159819:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828749.159822:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828749.159827:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787828749.159831:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828749.159833:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787828749.159838:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298222103 00000100:00000001:2.0:1787828749.159838:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828749.159865:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828749.159867:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298223360 00000100:00000001:0.0:1787828749.159872:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298222103 00000100:00000001:0.0:1787828749.159884:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298223360 00000100:00000001:0.0:1787828749.159887:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787828749.159889:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828749.159891:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787828749.159897:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828749/real 1787828749] req@ffff8d30c9e2fc00 x1874670646760960/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828774 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787828749.159906:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787828749.159907:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787828749.159910:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787828749.159913:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787828749.159914:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828749.159916:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828749.159918:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787828749.159922:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2fc00 x1874670646760960/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828774 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787828749.159928:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828749.159930:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787828749.159932:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828749.159933:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828749.159934:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828749.159936:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787828749.159949:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787828749.159951:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828749.159954:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787828749.159955:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787828749.159958:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646760960:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787828749.159962:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1787828749.159963:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787828749.159965:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828749.159967:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787828749.159969:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787828749.159971:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828749.159974:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2d2c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646761088:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787828749.159977:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787828749.159978:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787828749.159980:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828749.159981:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1787828749.159984:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828749.159989:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787828749.159990:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417873088 : -126234291678528 : ffff8d30c9e2d2c0) 00000100:00000001:0.0:1787828749.159994:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1787828749.160016:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:0.0:1787828749.160020:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1787828749.160022:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828749.160024:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828749.160025:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787828749.160028:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828749.160031:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298222103 00000100:00000001:0.0:1787828749.160033:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298223360 00000100:00000001:0.0:1787828749.160035:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298222103 00000100:00000001:0.0:1787828749.160038:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298223360 00000100:00000001:0.0:1787828749.160040:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787828749.160042:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828749.160043:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787828749.160046:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828749/real 1787828749] req@ffff8d30c9e2d2c0 x1874670646761088/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828774 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787828749.160053:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787828749.160054:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787828749.160056:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787828749.160057:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787828749.160058:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828749.160061:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828749.160062:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787828749.160068:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2d2c0 x1874670646761088/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828774 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787828749.160089:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828749.160091:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787828749.160095:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828749.160097:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828749.160099:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828749.160101:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787828749.160103:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787828749.160107:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828749.160112:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787828749.160116:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787828749.160121:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2d2c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646761088:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787828749.160127:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828749.160130:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828749.160132:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787828749.160134:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787828749.160138:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787828749.160140:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787828749.160142:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787828749.160145:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787828749.160147:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787828749.160150:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787828749.160152:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828749.160153:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828749.160155:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787828749.160156:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787828749.160158:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787828749.160159:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787828749.160160:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787828749.160161:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787828749.160162:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787828749.160164:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787828749.160165:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828749.160166:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828749.160169:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787828749.160171:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828749.160172:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787828749.160174:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787828749.160175:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1787828749.160176:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787828749.160177:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828749.160180:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787828749.160181:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787828749.160195:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828749.160198:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646761216:192.168.203.143@tcp:8:kworker.0 00000100:00000001:0.0:1787828749.160201:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787828749.160203:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787828749.160205:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828749.160207:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1787828749.160209:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828749.160215:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787828749.160216:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417871168 : -126234291680448 : ffff8d30c9e2cb40) 00000100:00000001:0.0:1787828749.160219:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1787828749.160231:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:0.0:1787828749.160235:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1787828749.160237:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828749.160238:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828749.160240:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787828749.160242:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828749.160246:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298222116 00000100:00000001:0.0:1787828749.160249:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298223360 00000100:00000001:0.0:1787828749.160252:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298222116 00000100:00000001:0.0:1787828749.160255:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298223360 00000100:00000001:0.0:1787828749.160258:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787828749.160261:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828749.160264:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787828749.160269:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828749/real 1787828749] req@ffff8d30c9e2cb40 x1874670646761216/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828774 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787828749.160281:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787828749.160283:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787828749.160287:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787828749.160289:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787828749.160292:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828749.160294:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828749.160297:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787828749.160303:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2cb40 x1874670646761216/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828774 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787828749.160313:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828749.160316:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787828749.160321:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828749.160322:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828749.160324:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828749.160326:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787828749.160327:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787828749.160330:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828749.160333:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787828749.160335:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787828749.160337:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646761216:192.168.203.143@tcp:8:kworker.0 00000100:00000001:0.0:1787828749.160341:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828749.160343:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828749.160344:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787828749.160346:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787828749.160348:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787828749.160349:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787828749.160350:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787828749.160352:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787828749.160354:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787828749.160356:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787828749.160358:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828749.160359:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828749.160362:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787828749.160364:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828749.160366:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787828749.160367:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828750.054772:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828750.054780:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828750.054891:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828750.054895:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828750.054898:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828751.078697:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828751.078702:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828751.078753:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828751.078767:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828751.078769:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828752.102739:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828752.102745:0:815:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1787828752.102748:0:815:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1787828752.102751:0:815:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828752.102755:0:815:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1787828752.102757:0:815:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828752.102759:0:815:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828752.102761:0:815:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828752.102848:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828752.102854:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828752.102875:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828753.126712:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828753.126717:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828753.126783:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828753.126787:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828753.126790:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828754.150731:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828754.150737:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828754.150796:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828754.150800:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828754.150803:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787828754.278719:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828754.278735:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1787828754.278741:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787828754.278747:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787828754.278755:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787828754.278766:0:468:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 3554 00000100:00080000:2.0:1787828754.278781:0:468:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787828754.278787:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828754.278792:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787828754.278799:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787828754.278804:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787828754.278831:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:2.0:1787828754.278838:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:2.0:1787828754.278842:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828754.278845:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:2.0:1787828754.278869:0:468:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787828754.278873:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828754.278876:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787828754.278880:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828754.278884:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828754.278888:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:2.0:1787828754.278891:0:468:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828754.278902:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787828754.278905:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828754.278909:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787828754.278912:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787828754.278916:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828754.278928:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787828754.278940:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2e1c0 x1874670646761344/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787828754.278953:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787828754.278958:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828754.278975:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787828754.278979:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828754.278983:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1787828754.278986:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787828754.278988:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787828754.278993:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787828754.278997:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3554 00000100:00080000:2.0:1787828754.279003:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787828754.279006:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828754.279008:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787828754.279012:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:1.0:1787828754.279013:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:2.0:1787828754.279014:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:1.0:1787828754.279019:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000020:00000001:2.0:1787828754.279019:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:1.0:1787828754.279024:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787828754.279027:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787828754.279029:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1787828754.279033:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 02000000:00000001:1.0:1787828754.279035:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787828754.279041:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787828754.279041:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828754.279046:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 02000000:00000001:1.0:1787828754.279047:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787828754.279056:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00100000:1.0:1787828754.279058:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646761344:192.168.203.143@tcp:250:kworker.0 00000100:00000001:2.0:1787828754.279064:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828754.279067:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787828754.279070:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1787828754.279073:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787828754.279078:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828754.279078:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787828754.279086:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1787828754.279087:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828754.279091:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 02000000:00000001:1.0:1787828754.279093:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000008:00100000:2.0:1787828754.279097:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1787828754.279103:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828754.279106:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828754.279112:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 02000000:00000001:2.0:1787828754.279113:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:1.0:1787828754.279117:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417876928 : -126234291674688 : ffff8d30c9e2e1c0) 02000000:00000001:2.0:1787828754.279118:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828754.279124:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:2.0:1787828754.279124:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787828754.279126:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787828754.279131:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828754.279138:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787828754.279145:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2d680 x1874670646761472/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787828754.279155:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787828754.279158:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828754.279162:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:1.0:1787828754.279164:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00080000:2.0:1787828754.279168:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1787828754.279174:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00080000:2.0:1787828754.279174:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1787828754.279178:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:1.0:1787828754.279181:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00080000:2.0:1787828754.279183:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787828754.279185:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828754.279188:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1787828754.279191:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:1.0:1787828754.279194:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787828754.279199:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3554 00000100:00000001:1.0:1787828754.279201:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4298228480 00000100:00080000:2.0:1787828754.279206:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1787828754.279210:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 00000100:00000001:2.0:1787828754.279211:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828754.279216:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:1.0:1787828754.279218:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4298228480 00000020:00000001:2.0:1787828754.279223:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:1.0:1787828754.279225:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 00000020:00000001:2.0:1787828754.279228:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:1.0:1787828754.279229:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000020:00000001:2.0:1787828754.279232:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:1.0:1787828754.279234:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1787828754.279239:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:1.0:1787828754.279240:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00000001:2.0:1787828754.279244:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828754.279248:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:2.0:1787828754.279254:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787828754.279257:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787828754.279261:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828754/real 1787828754] req@ffff8d30c9e2e1c0 x1874670646761344/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828779 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1787828754.279262:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787828754.279268:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828754.279275:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828754.279282:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000400:00000001:1.0:1787828754.279283:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000008:00100000:2.0:1787828754.279285:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000400:00000001:1.0:1787828754.279288:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000008:00000001:2.0:1787828754.279291:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828754.279294:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1787828754.279294:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828754.279300:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 02000000:00000001:2.0:1787828754.279302:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:1.0:1787828754.279304:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:2.0:1787828754.279305:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828754.279311:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828754.279312:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787828754.279316:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:1.0:1787828754.279317:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:2.0:1787828754.279319:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787828754.279326:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2e1c0 x1874670646761344/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828779 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787828754.279327:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787828754.279336:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2cb40 x1874670646761600/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787828754.279347:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828754.279350:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787828754.279353:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787828754.279354:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828754.279359:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828754.279361:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:2.0:1787828754.279363:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1787828754.279367:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00080000:2.0:1787828754.279368:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1787828754.279371:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828754.279372:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:1.0:1787828754.279375:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00080000:2.0:1787828754.279375:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787828754.279378:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787828754.279380:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1787828754.279384:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828754.279385:0:468:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3554 00000100:00080000:1.0:1787828754.279390:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00080000:2.0:1787828754.279392:0:468:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1787828754.279393:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000001:2.0:1787828754.279396:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00100000:1.0:1787828754.279400:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646761344:192.168.203.143@tcp:250:kworker.0 00000100:00000001:2.0:1787828754.279400:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787828754.279407:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:1.0:1787828754.279409:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:2.0:1787828754.279409:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787828754.279412:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:1.0:1787828754.279413:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000020:00000001:2.0:1787828754.279417:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:1.0:1787828754.279418:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787828754.279422:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 00000100:00000001:2.0:1787828754.279423:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828754.279427:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 02000000:00000001:1.0:1787828754.279429:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00080000:2.0:1787828754.279431:0:468:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000020:00000001:1.0:1787828754.279433:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787828754.279436:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 00000100:00000001:2.0:1787828754.279437:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828754.279440:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1787828754.279441:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787828754.279443:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 02000000:00000001:2.0:1787828754.279443:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828754.279448:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828754.279449:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787828754.279450:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:1.0:1787828754.279453:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000008:00100000:2.0:1787828754.279454:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00000001:1.0:1787828754.279457:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828754.279463:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787828754.279465:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:2.0:1787828754.279467:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828754.279468:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787828754.279471:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828754.279472:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1787828754.279475:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787828754.279478:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787828754.279479:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828754.279479:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787828754.279483:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828754.279484:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787828754.279488:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787828754.279488:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1787828754.279493:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828754.279494:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00100000:1.0:1787828754.279497:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2d680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646761472:192.168.203.143@tcp:38:kworker.0 02000000:00000001:2.0:1787828754.279499:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828754.279501:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787828754.279505:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787828754.279507:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828754.279510:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:2.0:1787828754.279510:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828754.279513:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828754.279521:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00100000:2.0:1787828754.279521:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2d2c0 x1874670646761728/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787828754.279524:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417874048 : -126234291677568 : ffff8d30c9e2d680) 00000100:00000001:1.0:1787828754.279531:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:2.0:1787828754.279539:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787828754.279543:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:1.0:1787828754.279546:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:2.0:1787828754.279551:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828754.279552:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787828754.279557:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787828754.279559:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828754.279561:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787828754.279583:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828754.279588:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298227222 00000100:00000001:1.0:1787828754.279591:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298228480 00000100:00000001:1.0:1787828754.279595:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298227222 00000100:00000001:1.0:1787828754.279598:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298228480 00000100:00000001:1.0:1787828754.279602:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787828754.279605:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828754.279609:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787828754.279651:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828754/real 1787828754] req@ffff8d30c9e2d680 x1874670646761472/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828779 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787828754.279669:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787828754.279671:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787828754.279676:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787828754.279680:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787828754.279682:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828754.279687:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787828754.279690:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787828754.279697:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2d680 x1874670646761472/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828779 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787828754.279705:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828754.279710:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787828754.279713:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828754.279715:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787828754.279716:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828754.279718:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787828754.279720:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787828754.279722:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828754.279726:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787828754.279728:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787828754.279731:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2d680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646761472:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787828754.279736:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787828754.279737:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787828754.279739:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828754.279741:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787828754.279743:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787828754.279745:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787828754.279748:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646761600:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787828754.279751:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787828754.279753:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787828754.279756:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828754.279758:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787828754.279761:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828754.279767:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787828754.279768:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417871168 : -126234291680448 : ffff8d30c9e2cb40) 00000100:00000001:1.0:1787828754.279772:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1787828754.279786:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787828754.279791:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787828754.279793:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787828754.279795:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828754.279796:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787828754.279799:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828754.279803:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298227223 00000100:00000001:1.0:1787828754.279829:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298228480 00000100:00000001:1.0:1787828754.279833:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298227223 00000100:00000001:1.0:1787828754.279835:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298228480 00000100:00000001:1.0:1787828754.279838:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787828754.279840:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828754.279842:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787828754.279847:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828754/real 1787828754] req@ffff8d30c9e2cb40 x1874670646761600/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828779 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787828754.279871:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787828754.279872:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787828754.279874:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787828754.279875:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787828754.279876:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828754.279877:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787828754.279878:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787828754.279881:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2cb40 x1874670646761600/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828779 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787828754.279886:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828754.279887:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787828754.279890:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828754.279891:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787828754.279892:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828754.279893:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787828754.279894:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787828754.279896:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828754.279898:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787828754.279899:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787828754.279901:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646761600:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787828754.279903:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828754.279905:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787828754.279906:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787828754.279907:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787828754.279909:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787828754.279910:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787828754.279911:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787828754.279912:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787828754.279913:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787828754.279915:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787828754.279916:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828754.279917:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787828754.279918:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787828754.279919:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787828754.279921:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787828754.279921:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787828754.279922:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787828754.279923:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787828754.279924:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787828754.279926:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787828754.279927:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828754.279928:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828754.279930:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787828754.279931:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828754.279932:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787828754.279933:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1787828754.279934:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787828754.279935:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787828754.279936:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828754.279938:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787828754.279939:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787828754.279941:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787828754.279943:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2d2c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646761728:192.168.203.143@tcp:8:kworker.0 00000100:00000001:1.0:1787828754.279945:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787828754.279946:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787828754.279947:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828754.279948:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787828754.279951:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828754.279955:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787828754.279956:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417873088 : -126234291678528 : ffff8d30c9e2d2c0) 00000100:00000001:1.0:1787828754.279958:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1787828754.279967:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787828754.279970:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787828754.279971:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787828754.279972:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828754.279973:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787828754.279976:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828754.279978:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298227236 00000100:00000001:1.0:1787828754.279980:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298228480 00000100:00000001:1.0:1787828754.279981:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298227236 00000100:00000001:1.0:1787828754.279983:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298228480 00000100:00000001:1.0:1787828754.279985:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787828754.279986:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828754.279987:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787828754.279990:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828754/real 1787828754] req@ffff8d30c9e2d2c0 x1874670646761728/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828779 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787828754.279994:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787828754.279995:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787828754.279997:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787828754.279998:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787828754.279999:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828754.280001:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787828754.280002:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787828754.280004:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2d2c0 x1874670646761728/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828779 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787828754.280009:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828754.280010:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787828754.280012:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828754.280013:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787828754.280014:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828754.280015:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787828754.280016:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787828754.280018:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828754.280019:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787828754.280021:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787828754.280023:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2d2c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646761728:192.168.203.143@tcp:8:kworker.0 00000100:00000001:1.0:1787828754.280025:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828754.280026:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787828754.280027:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787828754.280028:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787828754.280029:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787828754.280030:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787828754.280030:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787828754.280032:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787828754.280033:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787828754.280034:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787828754.280035:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828754.280036:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828754.280038:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787828754.280039:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828754.280040:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787828754.280041:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828755.174737:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828755.174745:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828755.174836:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828755.174841:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828755.174846:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828756.198777:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828756.198786:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828756.198901:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828756.198905:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828756.198908:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828757.222664:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828757.222669:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828757.222750:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828757.222753:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828757.222756:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828758.246737:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828758.246744:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828758.246870:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828758.246874:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828758.246877:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828759.270716:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828759.270722:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828759.270791:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828759.270795:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828759.270799:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787828759.398681:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828759.398687:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1787828759.398689:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787828759.398692:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787828759.398695:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787828759.398700:0:468:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 3559 00000100:00080000:2.0:1787828759.398708:0:468:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787828759.398710:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828759.398712:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787828759.398715:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787828759.398717:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787828759.398719:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:2.0:1787828759.398721:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:2.0:1787828759.398723:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828759.398724:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:2.0:1787828759.398727:0:468:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787828759.398728:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828759.398731:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787828759.398733:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828759.398735:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828759.398737:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:2.0:1787828759.398739:0:468:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828759.398746:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787828759.398748:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828759.398750:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787828759.398752:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787828759.398755:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828759.398764:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787828759.398782:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2fc00 x1874670646761856/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787828759.398791:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787828759.398795:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828759.398822:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787828759.398825:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828759.398828:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1787828759.398829:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787828759.398831:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787828759.398833:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787828759.398836:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3559 00000100:00080000:2.0:1787828759.398839:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787828759.398841:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828759.398842:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787828759.398844:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787828759.398846:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787828759.398848:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000020:00000001:2.0:1787828759.398850:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:2.0:1787828759.398851:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828759.398852:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:1.0:1787828759.398856:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00080000:2.0:1787828759.398856:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787828759.398860:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828759.398862:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1787828759.398864:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1787828759.398894:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 02000000:00000001:2.0:1787828759.398895:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787828759.398898:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 02000000:00000001:2.0:1787828759.398899:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828759.398903:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828759.398903:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1787828759.398908:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 02000000:00000001:1.0:1787828759.398909:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787828759.398913:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000008:00000001:2.0:1787828759.398914:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828759.398917:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828759.398919:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787828759.398923:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646761856:192.168.203.143@tcp:250:kworker.0 02000000:00000001:2.0:1787828759.398926:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787828759.398930:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828759.398932:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:2.0:1787828759.398934:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787828759.398937:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1787828759.398938:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1787828759.398940:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828759.398943:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828759.398947:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828759.398948:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787828759.398957:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787828759.398958:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2de00 x1874670646761984/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787828759.398969:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787828759.398971:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828759.398973:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828759.398974:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00080000:2.0:1787828759.398976:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1787828759.398978:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417883648 : -126234291667968 : ffff8d30c9e2fc00) 00000100:00080000:2.0:1787828759.398981:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:1.0:1787828759.398983:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:2.0:1787828759.398984:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787828759.398987:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787828759.398989:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787828759.398991:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3559 00000100:00080000:2.0:1787828759.398995:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787828759.398996:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828759.398997:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787828759.398999:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787828759.399000:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787828759.399002:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000020:00000001:2.0:1787828759.399003:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:2.0:1787828759.399005:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828759.399006:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:2.0:1787828759.399008:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787828759.399010:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828759.399011:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787828759.399014:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828759.399016:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828759.399017:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1787828759.399019:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1787828759.399021:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828759.399023:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828759.399026:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787828759.399027:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828759.399029:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:1.0:1787828759.399030:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828759.399030:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787828759.399034:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828759.399036:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4298233600 00000100:00000001:2.0:1787828759.399040:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828759.399045:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 00000100:00100000:2.0:1787828759.399045:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2c3c0 x1874670646762112/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787828759.399053:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787828759.399055:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828759.399056:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787828759.399058:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1787828759.399060:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4298233600 00000100:00080000:2.0:1787828759.399061:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1787828759.399063:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787828759.399066:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787828759.399075:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:1.0:1787828759.399077:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 00000100:00080000:2.0:1787828759.399078:0:468:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3559 00000100:00000001:1.0:1787828759.399084:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00080000:2.0:1787828759.399084:0:468:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787828759.399086:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787828759.399089:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828759.399089:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787828759.399095:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:1.0:1787828759.399096:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1787828759.399096:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787828759.399100:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:1.0:1787828759.399101:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:2.0:1787828759.399104:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:2.0:1787828759.399106:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828759.399107:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:2.0:1787828759.399109:0:468:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:1.0:1787828759.399110:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1787828759.399113:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828759.399114:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 02000000:00000001:2.0:1787828759.399117:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1787828759.399118:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:2.0:1787828759.399120:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787828759.399121:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1787828759.399124:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 02000000:00000001:2.0:1787828759.399125:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828759.399127:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787828759.399128:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:1.0:1787828759.399130:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828759.399133:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787828759.399141:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000008:00100000:2.0:1787828759.399141:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000008:00000001:2.0:1787828759.399146:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828759.399147:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828759.399149:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787828759.399154:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646761984:192.168.203.143@tcp:38:kworker.0 02000000:00000001:2.0:1787828759.399155:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787828759.399158:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828759.399160:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:2.0:1787828759.399160:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1787828759.399164:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1787828759.399164:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1787828759.399167:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828759.399169:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828759.399170:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787828759.399175:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828759.399178:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828759.399182:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787828759.399185:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417875968 : -126234291675648 : ffff8d30c9e2de00) 00000100:00100000:2.0:1787828759.399187:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2c780 x1874670646762240/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787828759.399190:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:2.0:1787828759.399198:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787828759.399201:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828759.399202:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828759.399208:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298232342 00000100:00000001:2.0:1787828759.399210:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828759.399213:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298233600 00000100:00000001:1.0:1787828759.399216:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298232342 00000100:00000001:1.0:1787828759.399218:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298233600 00000100:00000001:1.0:1787828759.399221:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787828759.399222:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828759.399224:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787828759.399225:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787828759.399226:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828759.399228:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787828759.399229:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787828759.399231:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787828759.399235:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646762112:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787828759.399237:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787828759.399239:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787828759.399240:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828759.399241:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787828759.399243:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828759.399246:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787828759.399247:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417869248 : -126234291682368 : ffff8d30c9e2c3c0) 00000100:00000001:1.0:1787828759.399249:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:1.0:1787828759.399254:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828759.399256:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298232342 00000100:00000001:1.0:1787828759.399258:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298233600 00000100:00000001:1.0:1787828759.399259:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298232342 00000100:00000001:1.0:1787828759.399261:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298233600 00000100:00000001:1.0:1787828759.399263:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787828759.399264:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828759.399265:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828759.399267:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828759.399268:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787828759.399269:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:1.0:1787828759.399271:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787828759.399271:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1787828759.399272:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787828759.399273:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787828759.399274:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828759.399275:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787828759.399276:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787828759.399278:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787828759.399280:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646762240:192.168.203.143@tcp:8:kworker.0 00000100:00000001:1.0:1787828759.399281:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787828759.399282:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787828759.399283:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828759.399284:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787828759.399286:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828759.399289:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787828759.399289:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417870208 : -126234291681408 : ffff8d30c9e2c780) 00000100:00000001:1.0:1787828759.399291:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:1.0:1787828759.399296:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828759.399298:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298232355 00000100:00000001:1.0:1787828759.399299:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298233600 00000100:00000001:1.0:1787828759.399301:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298232355 00000100:00000001:1.0:1787828759.399302:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298233600 00000100:00000001:1.0:1787828759.399303:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787828759.399304:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828759.399306:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828759.399307:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828759.399309:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787828759.399310:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:1.0:1787828759.399311:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787828759.399312:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1787828759.399313:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828759.399314:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828760.294689:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828760.294694:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828760.294741:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828760.294744:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828760.294747:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828761.318662:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828761.318668:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828761.318709:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828761.318712:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828761.318715:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828762.342711:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828762.342719:0:815:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1787828762.342723:0:815:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1787828762.342726:0:815:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828762.342732:0:815:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1787828762.342735:0:815:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828762.342737:0:815:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828762.342740:0:815:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828762.342827:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828762.342830:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828762.342833:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:1.0:1787828762.471030:0:2378:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.143:988 00000400:00000100:1.0:1787828762.471078:0:2378:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.143@tcp at host 192.168.203.143 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1787828762.471094:0:2378:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.143@tcp 00000100:00000001:1.0:1787828762.471106:0:2378:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787828762.471123:0:2378:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787828762.471127:0:2378:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828762.471130:0:2378:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:1.0:1787828762.471142:0:2378:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.143@tcp 00000100:00000001:1.0:1787828762.471149:0:2378:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787828762.471155:0:2378:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787828762.471158:0:2378:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828762.471161:0:2378:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:1.0:1787828762.471170:0:2378:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.143@tcp 00000100:00000001:2.0:1787828762.471171:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787828762.471177:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1787828762.471178:0:2378:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1787828762.471182:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00000001:1.0:1787828762.471186:0:2378:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787828762.471190:0:2378:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828762.471193:0:2378:0:(events.c:68:request_out_callback()) Process leaving 00000100:00100000:2.0:1787828762.471198:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828759/real 1787828762] req@ffff8d30c9e2de00 x1874670646761984/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828784 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000100:1.0:1787828762.471207:0:2378:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.143@tcp 00000100:00000001:1.0:1787828762.471219:0:2378:0:(events.c:34:request_out_callback()) Process entered 00000400:00000001:2.0:1787828762.471231:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000001:1.0:1787828762.471232:0:2378:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000400:00000001:2.0:1787828762.471235:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787828762.471238:0:2378:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828762.471243:0:2378:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1787828762.471245:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1787828762.471250:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787828762.471252:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828762.471255:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828762.471257:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787828762.471263:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2de00 x1874670646761984/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828784 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787828762.471270:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828762.471273:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787828762.471276:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828762.471278:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828762.471279:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828762.471282:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787828762.471284:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787828762.471287:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828762.471291:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:2.0:1787828762.471293:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1787828762.471296:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646761984:192.168.203.143@tcp:38:kworker.0 00000100:00000001:2.0:1787828762.471301:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1787828762.471304:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828759/real 1787828762] req@ffff8d30c9e2c3c0 x1874670646762112/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828784 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1787828762.471311:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787828762.471312:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1787828762.471315:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1787828762.471316:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787828762.471317:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828762.471318:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828762.471320:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787828762.471325:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2c3c0 x1874670646762112/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828784 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787828762.471335:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828762.471345:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787828762.471348:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828762.471350:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828762.471352:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828762.471354:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787828762.471356:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787828762.471360:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828762.471365:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:2.0:1787828762.471368:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1787828762.471373:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646762112:192.168.203.143@tcp:38:kworker.0 00000100:00000001:2.0:1787828762.471378:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1787828762.471383:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828759/real 1787828762] req@ffff8d30c9e2fc00 x1874670646761856/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828784 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1787828762.471395:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787828762.471397:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1787828762.471402:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1787828762.471404:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787828762.471406:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828762.471408:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828762.471410:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787828762.471416:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2fc00 x1874670646761856/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828784 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787828762.471424:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828762.471427:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787828762.471430:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828762.471433:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828762.471435:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828762.471437:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787828762.471439:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787828762.471443:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828762.471447:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:2.0:1787828762.471450:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1787828762.471454:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646761856:192.168.203.143@tcp:250:kworker.0 00000100:00000001:2.0:1787828762.471459:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828762.471461:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828762.471463:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787828762.471467:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1787828762.471473:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787828762.471477:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:2.0:1787828762.471479:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:2.0:1787828762.471483:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787828762.471485:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1787828762.471492:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787828762.471494:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828762.471496:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828762.471498:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787828762.471499:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1787828762.471501:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787828762.471503:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:2.0:1787828762.471504:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:2.0:1787828762.471506:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787828762.471507:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1787828762.471511:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787828762.471512:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828762.471514:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828762.471516:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787828762.471517:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1787828762.471520:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787828762.471522:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:2.0:1787828762.471523:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:2.0:1787828762.471524:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787828762.471526:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1787828762.471528:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787828762.471531:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828762.471534:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828762.471536:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787828762.471538:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1787828762.471540:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1787828762.471546:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828759/real 1787828762] req@ffff8d30c9e2c780 x1874670646762240/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828784 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1787828762.471556:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787828762.471558:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1787828762.471562:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1787828762.471564:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787828762.471566:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828762.471568:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828762.471570:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787828762.471575:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2c780 x1874670646762240/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828784 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787828762.471583:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828762.471586:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787828762.471589:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828762.471591:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828762.471593:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828762.471595:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787828762.471597:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787828762.471632:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828762.471639:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:2.0:1787828762.471642:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1787828762.471647:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646762240:192.168.203.143@tcp:8:kworker.0 00000100:00000001:2.0:1787828762.471653:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828762.471655:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828762.471657:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787828762.471659:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1787828762.471663:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787828762.471665:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:2.0:1787828762.471667:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:2.0:1787828762.471670:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787828762.471672:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1787828762.471674:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787828762.471675:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828762.471678:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828762.471687:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1787828762.471689:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828762.471693:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787828762.471695:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828763.366735:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828763.366741:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828763.366865:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828763.366874:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828763.366880:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828764.390713:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828764.390719:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828764.390800:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828764.390808:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828764.390814:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787828764.518799:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828764.518813:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1787828764.518817:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787828764.518822:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787828764.518829:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787828764.518847:0:468:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 3565 00000100:00080000:0.0:1787828764.518862:0:468:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787828764.518866:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787828764.518870:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787828764.518876:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787828764.518880:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787828764.518884:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:0.0:1787828764.518887:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:0.0:1787828764.518889:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787828764.518890:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:0.0:1787828764.518896:0:468:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787828764.518899:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828764.518903:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787828764.518908:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787828764.518912:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828764.518914:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1787828764.518917:0:468:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828764.518931:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787828764.518933:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828764.518946:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787828764.518949:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787828764.518955:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828764.518971:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828764.518984:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d25a40 x1874670646762368/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787828764.518998:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787828764.519004:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828764.519024:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787828764.519028:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828764.519032:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1787828764.519034:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787828764.519035:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787828764.519039:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787828764.519041:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3565 00000100:00080000:0.0:1787828764.519046:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787828764.519050:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787828764.519053:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787828764.519057:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787828764.519060:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787828764.519064:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000020:00000001:0.0:1787828764.519068:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:0.0:1787828764.519073:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787828764.519076:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:0.0:1787828764.519083:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787828764.519091:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828764.519092:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:0.0:1787828764.519096:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:2.0:1787828764.519107:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 02000000:00000001:0.0:1787828764.519110:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787828764.519114:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 02000000:00000001:0.0:1787828764.519120:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828764.519122:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787828764.519128:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828764.519129:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1787828764.519140:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1787828764.519149:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828764.519153:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:0.0:1787828764.519154:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828764.519161:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828764.519167:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828764.519169:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787828764.519174:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787828764.519180:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d25a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646762368:192.168.203.143@tcp:250:kworker.0 00000100:00000001:0.0:1787828764.519181:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:2.0:1787828764.519194:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787828764.519201:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787828764.519202:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787828764.519208:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828764.519210:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828764.519217:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:0.0:1787828764.519227:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828764.519235:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828764.519247:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d24f00 x1874670646762496/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787828764.519265:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787828764.519266:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787828764.519285:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828764.519290:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601343040 : -126234108208576 : ffff8d30d4d25a40) 00000100:00000001:0.0:1787828764.519296:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828764.519299:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00080000:0.0:1787828764.519305:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828764.519313:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1787828764.519315:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787828764.519318:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787828764.519322:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787828764.519328:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3565 00000100:00080000:0.0:1787828764.519337:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787828764.519341:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787828764.519354:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787828764.519359:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787828764.519363:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787828764.519368:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000800:00000100:2.0:1787828764.519372:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000020:00000001:0.0:1787828764.519378:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:0.0:1787828764.519385:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787828764.519387:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:0.0:1787828764.519396:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787828764.519398:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1787828764.519402:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828764.519408:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 02000000:00000001:0.0:1787828764.519409:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:2.0:1787828764.519416:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828764.519420:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787828764.519427:0:2390:0:(events.c:68:request_out_callback()) Process leaving 02000000:00000001:0.0:1787828764.519430:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828764.519436:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:2.0:1787828764.519440:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000008:00100000:0.0:1787828764.519441:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1787828764.519452:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828764.519454:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4298238720 00000100:00000001:0.0:1787828764.519457:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828764.519470:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:2.0:1787828764.519471:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 02000000:00000001:0.0:1787828764.519473:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828764.519480:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787828764.519483:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:2.0:1787828764.519486:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4298238720 02000000:00000001:0.0:1787828764.519487:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828764.519504:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1787828764.519505:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828764.519520:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00100000:0.0:1787828764.519524:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d24000 x1874670646762624/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787828764.519529:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828764.519539:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00000001:0.0:1787828764.519553:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00100000:2.0:1787828764.519558:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828764/real 1787828764] req@ffff8d30d4d25a40 x1874670646762368/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828789 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787828764.519560:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828764.519569:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787828764.519577:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828764.519595:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000400:00000001:2.0:1787828764.519635:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787828764.519642:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1787828764.519648:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1787828764.519652:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787828764.519655:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828764.519659:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828764.519662:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787828764.519672:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d25a40 x1874670646762368/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828789 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787828764.519682:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828764.519687:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:2.0:1787828764.519688:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787828764.519697:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787828764.519703:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828764.519707:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:2.0:1787828764.519711:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828764.519716:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:0.0:1787828764.519719:0:468:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3565 00000100:00000001:2.0:1787828764.519725:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00080000:0.0:1787828764.519730:0:468:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787828764.519731:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787828764.519754:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00000001:0.0:1787828764.519762:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787828764.519772:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:2.0:1787828764.519772:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000020:00000001:0.0:1787828764.519782:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:2.0:1787828764.519785:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000020:00000001:0.0:1787828764.519787:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787828764.519796:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00100000:2.0:1787828764.519802:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d25a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646762368:192.168.203.143@tcp:250:kworker.0 00000020:00000001:0.0:1787828764.519806:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:0.0:1787828764.519813:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787828764.519820:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:2.0:1787828764.519826:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828764.519841:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00080000:0.0:1787828764.519845:0:468:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787828764.519846:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787828764.519852:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 00000100:00000001:0.0:1787828764.519856:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828764.519860:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 02000000:00000001:0.0:1787828764.519863:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000020:00000001:2.0:1787828764.519867:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:2.0:1787828764.519873:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:2.0:1787828764.519878:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787828764.519881:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1787828764.519887:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787828764.519888:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828764.519892:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828764.519902:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1787828764.519905:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828764.519908:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787828764.519913:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 02000000:00000001:0.0:1787828764.519942:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787828764.519950:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828764.519955:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:2.0:1787828764.519955:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1787828764.519966:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000008:00100000:0.0:1787828764.519967:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00000001:2.0:1787828764.519969:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828764.519978:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000008:00000001:0.0:1787828764.519980:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828764.519985:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787828764.519988:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828764.519993:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828764.520003:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00100000:2.0:1787828764.520005:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d24f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646762496:192.168.203.143@tcp:38:kworker.0 02000000:00000001:0.0:1787828764.520009:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828764.520017:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:0.0:1787828764.520019:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787828764.520024:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787828764.520027:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787828764.520030:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828764.520033:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828764.520040:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:0.0:1787828764.520043:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828764.520047:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828764.520059:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d270c0 x1874670646762752/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787828764.520060:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787828764.520065:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601340160 : -126234108211456 : ffff8d30d4d24f00) 00000100:00000001:2.0:1787828764.520079:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:0.0:1787828764.520098:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787828764.520103:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828764.520111:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:2.0:1787828764.520118:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:2.0:1787828764.520132:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1787828764.520139:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828764.520141:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828764.520144:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1787828764.520149:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828764.520155:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298237463 00000100:00000001:2.0:1787828764.520159:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298238720 00000100:00000001:2.0:1787828764.520163:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298237463 00000100:00000001:2.0:1787828764.520166:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298238720 00000100:00000001:2.0:1787828764.520168:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787828764.520171:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828764.520173:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1787828764.520180:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828764/real 1787828764] req@ffff8d30d4d24f00 x1874670646762496/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828789 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1787828764.520190:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787828764.520192:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1787828764.520195:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1787828764.520197:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787828764.520198:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828764.520200:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828764.520202:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787828764.520206:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d24f00 x1874670646762496/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828789 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787828764.520214:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828764.520217:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787828764.520219:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828764.520221:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828764.520223:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828764.520225:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787828764.520227:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787828764.520230:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828764.520235:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:2.0:1787828764.520237:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1787828764.520241:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d24f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646762496:192.168.203.143@tcp:38:kworker.0 00000100:00000001:2.0:1787828764.520247:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1787828764.520249:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787828764.520251:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828764.520254:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787828764.520257:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828764.520259:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787828764.520263:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d24000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646762624:192.168.203.143@tcp:38:kworker.0 00000100:00000001:2.0:1787828764.520266:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787828764.520268:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1787828764.520270:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828764.520272:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1787828764.520276:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828764.520282:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787828764.520283:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601336320 : -126234108215296 : ffff8d30d4d24000) 00000100:00000001:2.0:1787828764.520286:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1787828764.520301:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:2.0:1787828764.520306:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1787828764.520308:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828764.520310:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828764.520311:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1787828764.520314:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828764.520318:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298237463 00000100:00000001:2.0:1787828764.520321:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298238720 00000100:00000001:2.0:1787828764.520323:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298237463 00000100:00000001:2.0:1787828764.520325:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298238720 00000100:00000001:2.0:1787828764.520329:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787828764.520331:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828764.520333:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1787828764.520339:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828764/real 1787828764] req@ffff8d30d4d24000 x1874670646762624/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828789 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1787828764.520348:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787828764.520350:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1787828764.520354:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1787828764.520357:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787828764.520358:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828764.520362:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828764.520364:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787828764.520369:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d24000 x1874670646762624/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828789 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787828764.520377:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828764.520380:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787828764.520382:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828764.520384:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828764.520386:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828764.520388:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787828764.520390:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787828764.520394:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828764.520398:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:2.0:1787828764.520400:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1787828764.520404:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d24000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646762624:192.168.203.143@tcp:38:kworker.0 00000100:00000001:2.0:1787828764.520407:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828764.520409:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828764.520411:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787828764.520413:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1787828764.520417:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787828764.520418:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:2.0:1787828764.520420:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:2.0:1787828764.520422:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787828764.520424:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1787828764.520427:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787828764.520429:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828764.520430:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828764.520431:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787828764.520432:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1787828764.520434:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787828764.520435:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:2.0:1787828764.520436:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:2.0:1787828764.520438:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787828764.520439:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1787828764.520441:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787828764.520443:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828764.520445:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828764.520449:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1787828764.520451:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828764.520453:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787828764.520455:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1787828764.520457:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1787828764.520458:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787828764.520460:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828764.520463:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787828764.520465:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828764.520467:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787828764.520470:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d270c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646762752:192.168.203.143@tcp:8:kworker.0 00000100:00000001:2.0:1787828764.520473:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787828764.520475:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1787828764.520477:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828764.520478:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1787828764.520480:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828764.520487:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787828764.520488:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601348800 : -126234108202816 : ffff8d30d4d270c0) 00000100:00000001:2.0:1787828764.520492:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1787828764.520507:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:2.0:1787828764.520511:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1787828764.520515:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828764.520516:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828764.520518:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1787828764.520520:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828764.520524:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298237476 00000100:00000001:2.0:1787828764.520526:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298238720 00000100:00000001:2.0:1787828764.520529:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298237476 00000100:00000001:2.0:1787828764.520531:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298238720 00000100:00000001:2.0:1787828764.520533:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787828764.520535:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828764.520537:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1787828764.520541:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828764/real 1787828764] req@ffff8d30d4d270c0 x1874670646762752/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828789 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1787828764.520550:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787828764.520551:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1787828764.520554:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1787828764.520555:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787828764.520557:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828764.520559:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828764.520560:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787828764.520564:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d270c0 x1874670646762752/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828789 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787828764.520570:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828764.520572:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787828764.520575:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828764.520577:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828764.520578:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828764.520580:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787828764.520581:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787828764.520584:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828764.520587:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:2.0:1787828764.520590:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1787828764.520593:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d270c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646762752:192.168.203.143@tcp:8:kworker.0 00000100:00000001:2.0:1787828764.520596:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828764.520598:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828764.520631:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787828764.520632:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1787828764.520635:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787828764.520636:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:2.0:1787828764.520637:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:2.0:1787828764.520640:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787828764.520641:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1787828764.520644:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787828764.520645:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828764.520648:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828764.520651:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1787828764.520652:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828764.520654:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787828764.520655:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828765.414698:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828765.414708:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828765.414768:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828765.414773:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828765.414777:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828766.438718:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828766.438760:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828766.438830:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828766.438835:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828766.438839:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828767.462739:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828767.462747:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828767.462800:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828767.462804:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828767.462808:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828768.486700:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828768.486706:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828768.486777:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828768.486780:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828768.486782:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828769.510724:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828769.510731:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828769.510805:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828769.510810:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828769.510814:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787828769.638804:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828769.638814:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1787828769.638817:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1787828769.638821:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787828769.638825:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1787828769.638832:0:40798:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 3570 00000100:00080000:1.0:1787828769.638843:0:40798:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1787828769.638846:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787828769.638848:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:1.0:1787828769.638852:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:1.0:1787828769.638855:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:1.0:1787828769.638858:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:1.0:1787828769.638860:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:1.0:1787828769.638861:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787828769.638862:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:1.0:1787828769.638866:0:40798:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:1.0:1787828769.638868:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828769.638870:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1787828769.638873:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787828769.638876:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828769.638878:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:1.0:1787828769.638880:0:40798:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828769.638893:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1787828769.638894:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828769.638898:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1787828769.638901:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1787828769.638905:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828769.638919:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787828769.638931:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d13ef0c0 x1874670646762880/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787828769.638943:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787828769.638949:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828769.638970:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1787828769.638973:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828769.638976:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1787828769.638977:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1787828769.638978:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787828769.638981:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1787828769.638983:0:40798:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3570 00000100:00080000:1.0:1787828769.638987:0:40798:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1787828769.638989:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787828769.638990:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:1.0:1787828769.638992:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:1.0:1787828769.638993:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:1.0:1787828769.638995:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000020:00000001:1.0:1787828769.638997:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:1.0:1787828769.638999:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787828769.639000:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:1.0:1787828769.639003:0:40798:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:1.0:1787828769.639004:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828769.639006:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1787828769.639011:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787828769.639013:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828769.639014:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:1.0:1787828769.639019:0:40798:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1787828769.639023:0:40798:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828769.639025:0:40798:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828769.639028:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1787828769.639029:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828769.639031:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1787828769.639033:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:2.0:1787828769.639035:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:1.0:1787828769.639036:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828769.639044:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1787828769.639045:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828769.639049:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00100000:1.0:1787828769.639054:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d13ede00 x1874670646763008/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787828769.639056:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787828769.639061:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828769.639069:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787828769.639072:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828769.639074:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:1.0:1787828769.639075:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1787828769.639080:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 02000000:00000001:2.0:1787828769.639081:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00080000:1.0:1787828769.639087:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:1.0:1787828769.639091:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 02000000:00000001:2.0:1787828769.639092:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1787828769.639094:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787828769.639100:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00100000:2.0:1787828769.639106:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d13ef0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646762880:192.168.203.143@tcp:250:kworker.0 00000100:00080000:1.0:1787828769.639107:0:40798:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3570 00000100:00000001:2.0:1787828769.639122:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00080000:1.0:1787828769.639123:0:40798:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 02000000:00000001:2.0:1787828769.639127:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:1.0:1787828769.639129:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 02000000:00000001:2.0:1787828769.639130:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828769.639134:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 02000000:00000001:2.0:1787828769.639135:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000020:00000001:1.0:1787828769.639142:0:40798:0:(genops.c:792:class_conn2export()) Process entered 02000000:00000001:2.0:1787828769.639144:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787828769.639145:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:1.0:1787828769.639150:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000020:00000001:1.0:1787828769.639154:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:1.0:1787828769.639157:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787828769.639158:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:2.0:1787828769.639163:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00080000:1.0:1787828769.639164:0:40798:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787828769.639169:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839541350592 : -126234168201024 : ffff8d30d13ef0c0) 00000100:00000001:1.0:1787828769.639171:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828769.639175:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:2.0:1787828769.639176:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 02000000:00000001:1.0:1787828769.639186:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787828769.639191:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828769.639192:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:1.0:1787828769.639194:0:40798:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1787828769.639197:0:40798:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828769.639199:0:40798:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828769.639202:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1787828769.639203:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828769.639205:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1787828769.639215:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1787828769.639216:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:2.0:1787828769.639223:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787828769.639229:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828769.639234:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1787828769.639240:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00100000:1.0:1787828769.639242:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d13eed00 x1874670646763136/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787828769.639243:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828769.639248:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1787828769.639256:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828769.639261:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787828769.639265:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4298243840 00000100:00000001:1.0:1787828769.639267:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828769.639271:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1787828769.639273:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:2.0:1787828769.639278:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 00000100:00080000:1.0:1787828769.639283:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1787828769.639286:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1787828769.639288:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787828769.639291:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1787828769.639294:0:40798:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3570 00000100:00080000:1.0:1787828769.639299:0:40798:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787828769.639300:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4298243840 00000100:00000001:1.0:1787828769.639302:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787828769.639304:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:1.0:1787828769.639308:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:1.0:1787828769.639310:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:1.0:1787828769.639313:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:2.0:1787828769.639313:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 00000020:00000001:1.0:1787828769.639323:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:2.0:1787828769.639327:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787828769.639331:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828769.639332:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828769.639335:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:2.0:1787828769.639337:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00080000:1.0:1787828769.639346:0:40798:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:1.0:1787828769.639348:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828769.639354:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00100000:2.0:1787828769.639355:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828769/real 1787828769] req@ffff8d30d13ef0c0 x1874670646762880/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828794 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:1.0:1787828769.639365:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787828769.639374:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828769.639379:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:1.0:1787828769.639383:0:40798:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000008:00000001:1.0:1787828769.639388:0:40798:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828769.639390:0:40798:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:2.0:1787828769.639391:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 02000000:00000001:1.0:1787828769.639396:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000400:00000001:2.0:1787828769.639396:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 02000000:00000001:1.0:1787828769.639402:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828769.639405:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1787828769.639407:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1787828769.639409:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828769.639409:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1787828769.639418:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787828769.639419:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828769.639422:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787828769.639428:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d13ef840 x1874670646763264/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787828769.639429:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828769.639434:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787828769.639446:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d13ef0c0 x1874670646762880/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828794 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787828769.639458:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787828769.639460:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828769.639465:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828769.639467:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787828769.639474:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828769.639475:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828769.639476:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828769.639478:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828769.639484:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787828769.639486:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787828769.639490:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828769.639492:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:2.0:1787828769.639494:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1787828769.639497:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d13ef0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646762880:192.168.203.143@tcp:250:kworker.0 00000100:00000001:2.0:1787828769.639503:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828769.639505:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828769.639507:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787828769.639509:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1787828769.639515:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787828769.639518:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:2.0:1787828769.639521:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:2.0:1787828769.639524:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787828769.639525:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1787828769.639527:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787828769.639528:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828769.639530:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828769.639536:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1787828769.639537:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828769.639539:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787828769.639540:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1787828769.639542:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1787828769.639543:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787828769.639545:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828769.639548:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787828769.639550:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828769.639552:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787828769.639555:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d13ede00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646763008:192.168.203.143@tcp:38:kworker.0 00000100:00000001:2.0:1787828769.639559:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787828769.639562:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1787828769.639563:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828769.639565:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1787828769.639568:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828769.639576:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787828769.639577:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839541345792 : -126234168205824 : ffff8d30d13ede00) 00000100:00000001:2.0:1787828769.639582:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1787828769.639597:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:2.0:1787828769.639635:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1787828769.639637:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828769.639639:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828769.639640:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1787828769.639642:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828769.639646:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298242583 00000100:00000001:2.0:1787828769.639648:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298243840 00000100:00000001:2.0:1787828769.639650:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298242583 00000100:00000001:2.0:1787828769.639652:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298243840 00000100:00000001:2.0:1787828769.639653:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787828769.639655:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828769.639656:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1787828769.639659:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828769/real 1787828769] req@ffff8d30d13ede00 x1874670646763008/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828794 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1787828769.639665:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787828769.639666:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1787828769.639668:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1787828769.639669:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787828769.639670:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828769.639671:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828769.639672:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787828769.639675:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d13ede00 x1874670646763008/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828794 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787828769.639679:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828769.639681:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787828769.639682:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828769.639683:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828769.639684:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828769.639685:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787828769.639686:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787828769.639688:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828769.639690:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:2.0:1787828769.639691:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1787828769.639693:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d13ede00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646763008:192.168.203.143@tcp:38:kworker.0 00000100:00000001:2.0:1787828769.639696:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1787828769.639697:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787828769.639698:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828769.639717:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787828769.639718:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828769.639720:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787828769.639722:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d13eed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646763136:192.168.203.143@tcp:38:kworker.0 00000100:00000001:2.0:1787828769.639725:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787828769.639726:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1787828769.639728:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828769.639729:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1787828769.639731:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828769.639735:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787828769.639736:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839541349632 : -126234168201984 : ffff8d30d13eed00) 00000100:00000001:2.0:1787828769.639739:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1787828769.639748:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:2.0:1787828769.639751:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1787828769.639752:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828769.639754:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828769.639755:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1787828769.639756:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828769.639759:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298242583 00000100:00000001:2.0:1787828769.639761:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298243840 00000100:00000001:2.0:1787828769.639762:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298242583 00000100:00000001:2.0:1787828769.639763:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298243840 00000100:00000001:2.0:1787828769.639765:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787828769.639766:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828769.639767:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1787828769.639770:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828769/real 1787828769] req@ffff8d30d13eed00 x1874670646763136/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828794 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1787828769.639775:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787828769.639775:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1787828769.639777:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1787828769.639778:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787828769.639779:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828769.639780:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828769.639781:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787828769.639784:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d13eed00 x1874670646763136/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828794 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787828769.639788:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828769.639790:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787828769.639791:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828769.639792:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828769.639793:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828769.639794:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787828769.639795:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787828769.639796:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828769.639798:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:2.0:1787828769.639799:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1787828769.639801:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d13eed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646763136:192.168.203.143@tcp:38:kworker.0 00000100:00000001:2.0:1787828769.639803:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1787828769.639804:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787828769.639805:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828769.639807:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787828769.639808:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828769.639809:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787828769.639811:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d13ef840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646763264:192.168.203.143@tcp:8:kworker.0 00000100:00000001:2.0:1787828769.639814:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787828769.639815:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1787828769.639816:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828769.639817:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1787828769.639818:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828769.639821:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787828769.639822:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839541352512 : -126234168199104 : ffff8d30d13ef840) 00000100:00000001:2.0:1787828769.639824:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1787828769.639830:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:2.0:1787828769.639833:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1787828769.639834:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828769.639835:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828769.639836:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1787828769.639837:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828769.639839:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298242596 00000100:00000001:2.0:1787828769.639841:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298243840 00000100:00000001:2.0:1787828769.639842:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298242596 00000100:00000001:2.0:1787828769.639844:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298243840 00000100:00000001:2.0:1787828769.639845:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787828769.639847:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828769.639848:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1787828769.639850:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828769/real 1787828769] req@ffff8d30d13ef840 x1874670646763264/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828794 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1787828769.639854:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787828769.639855:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1787828769.639856:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1787828769.639857:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787828769.639858:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828769.639859:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828769.639860:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787828769.639862:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d13ef840 x1874670646763264/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828794 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787828769.639866:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828769.639867:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787828769.639868:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828769.639870:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828769.639870:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828769.639872:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787828769.639873:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787828769.639874:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828769.639877:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:2.0:1787828769.639878:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1787828769.639880:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d13ef840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646763264:192.168.203.143@tcp:8:kworker.0 00000100:00000001:2.0:1787828769.639883:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828769.639884:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828769.639885:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787828769.639886:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1787828769.639887:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787828769.639888:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:2.0:1787828769.639889:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:2.0:1787828769.639890:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787828769.639891:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1787828769.639893:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787828769.639894:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828769.639895:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828769.639896:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787828769.639896:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1787828769.639898:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787828769.639898:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:2.0:1787828769.639899:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:2.0:1787828769.639901:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787828769.639902:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1787828769.639904:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787828769.639905:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828769.639907:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828769.639908:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787828769.639909:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1787828769.639911:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787828769.639912:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:2.0:1787828769.639913:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:2.0:1787828769.639915:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787828769.639916:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1787828769.639917:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787828769.639918:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828769.639920:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828769.639924:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1787828769.639925:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828769.639928:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787828769.639929:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828770.534690:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828770.534710:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828770.534776:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828770.534780:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828770.534783:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828771.558667:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828771.558674:0:815:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1787828771.558675:0:815:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1787828771.558678:0:815:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828771.558681:0:815:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1787828771.558683:0:815:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828771.558684:0:815:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828771.558698:0:815:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828771.558745:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828771.558748:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828771.558751:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828772.582736:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828772.582743:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828772.582796:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828772.582799:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828772.582802:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828773.606728:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828773.606732:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828773.606797:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828773.606808:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828773.606812:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828774.630719:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828774.630724:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828774.630785:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828774.630791:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828774.630796:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787828774.758824:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828774.758838:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1787828774.758842:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787828774.758846:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787828774.758853:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787828774.758863:0:40798:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 3575 00000100:00080000:0.0:1787828774.758877:0:40798:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787828774.758882:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787828774.758885:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787828774.758908:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787828774.758911:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787828774.758915:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:0.0:1787828774.758918:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:0.0:1787828774.758921:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787828774.758923:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:0.0:1787828774.758930:0:40798:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787828774.758933:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828774.758937:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787828774.758942:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787828774.758946:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828774.758949:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1787828774.758953:0:40798:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828774.758966:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787828774.758969:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828774.758973:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787828774.758976:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787828774.758982:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828774.758996:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828774.759010:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d27c00 x1874670646763392/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787828774.759025:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787828774.759031:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828774.759071:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787828774.759075:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828774.759080:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1787828774.759081:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787828774.759083:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787828774.759087:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787828774.759090:0:40798:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3575 00000100:00080000:0.0:1787828774.759096:0:40798:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787828774.759099:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787828774.759101:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787828774.759105:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787828774.759107:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787828774.759110:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000020:00000001:0.0:1787828774.759113:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:0.0:1787828774.759116:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787828774.759117:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:2.0:1787828774.759120:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00080000:0.0:1787828774.759122:0:40798:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787828774.759128:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828774.759129:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 02000000:00000001:0.0:1787828774.759131:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:2.0:1787828774.759134:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 02000000:00000001:0.0:1787828774.759139:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787828774.759140:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787828774.759144:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828774.759146:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828774.759150:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 02000000:00000001:2.0:1787828774.759153:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000008:00100000:0.0:1787828774.759154:0:40798:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1787828774.759159:0:40798:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828774.759159:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787828774.759163:0:40798:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828774.759168:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828774.759170:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787828774.759173:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828774.759176:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787828774.759178:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00100000:2.0:1787828774.759178:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d27c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646763392:192.168.203.143@tcp:250:kworker.0 02000000:00000001:0.0:1787828774.759182:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828774.759186:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:0.0:1787828774.759190:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828774.759192:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1787828774.759196:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828774.759199:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d252c0 x1874670646763520/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:2.0:1787828774.759200:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1787828774.759207:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828774.759209:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787828774.759212:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828774.759215:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787828774.759218:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828774.759221:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1787828774.759222:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787828774.759224:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:2.0:1787828774.759225:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601351680 : -126234108199936 : ffff8d30d4d27c00) 00000100:00080000:0.0:1787828774.759229:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787828774.759230:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:0.0:1787828774.759237:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787828774.759241:0:40798:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3575 00000100:00080000:0.0:1787828774.759247:0:40798:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787828774.759250:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787828774.759251:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787828774.759255:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787828774.759257:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787828774.759260:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000020:00000001:0.0:1787828774.759262:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:0.0:1787828774.759266:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000800:00000100:2.0:1787828774.759269:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:0.0:1787828774.759270:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:0.0:1787828774.759279:0:40798:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787828774.759283:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828774.759285:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:2.0:1787828774.759286:0:2390:0:(events.c:34:request_out_callback()) Process entered 02000000:00000001:0.0:1787828774.759290:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787828774.759293:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 02000000:00000001:0.0:1787828774.759295:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828774.759299:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828774.759300:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1787828774.759304:0:40798:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00000001:2.0:1787828774.759305:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000008:00000001:0.0:1787828774.759309:0:40798:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828774.759314:0:40798:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828774.759315:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828774.759319:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787828774.759323:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828774.759323:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4298248960 00000100:00000001:0.0:1787828774.759328:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787828774.759332:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787828774.759334:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828774.759339:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1787828774.759341:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828774.759350:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4298248960 00000100:00100000:0.0:1787828774.759351:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d25680 x1874670646763648/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787828774.759361:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1787828774.759366:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787828774.759370:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828774.759370:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787828774.759375:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828774.759377:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787828774.759380:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:2.0:1787828774.759384:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00080000:0.0:1787828774.759386:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1787828774.759389:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787828774.759392:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787828774.759397:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00100000:2.0:1787828774.759397:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828774/real 1787828774] req@ffff8d30d4d27c00 x1874670646763392/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828799 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828774.759402:0:40798:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3575 00000100:00080000:0.0:1787828774.759410:0:40798:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787828774.759415:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000001:2.0:1787828774.759416:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000001:0.0:1787828774.759417:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000400:00000001:2.0:1787828774.759420:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000020:00000001:0.0:1787828774.759423:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787828774.759426:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:2.0:1787828774.759426:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000020:00000001:0.0:1787828774.759429:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:2.0:1787828774.759431:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000020:00000001:0.0:1787828774.759435:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:2.0:1787828774.759435:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828774.759457:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828774.759458:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828774.759460:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828774.759464:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00100000:2.0:1787828774.759470:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d27c00 x1874670646763392/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828799 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828774.759472:0:40798:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787828774.759478:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828774.759484:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787828774.759489:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787828774.759490:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:0.0:1787828774.759496:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828774.759498:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787828774.759499:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1787828774.759505:0:40798:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00000001:2.0:1787828774.759505:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000008:00000001:0.0:1787828774.759512:0:40798:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828774.759513:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828774.759517:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828774.759519:0:40798:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828774.759523:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787828774.759524:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828774.759525:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787828774.759526:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787828774.759528:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828774.759533:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787828774.759537:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828774.759537:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787828774.759543:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:0.0:1787828774.759562:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d26940 x1874670646763776/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828774.759567:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:2.0:1787828774.759574:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000001:0.0:1787828774.759582:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00100000:2.0:1787828774.759583:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d27c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646763392:192.168.203.143@tcp:250:kworker.0 00000100:00000001:0.0:1787828774.759587:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828774.759593:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828774.759604:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828774.759608:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828774.759610:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787828774.759690:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1787828774.759696:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787828774.759700:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:2.0:1787828774.759702:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:2.0:1787828774.759707:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787828774.759710:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1787828774.759715:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787828774.759717:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828774.759720:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828774.759729:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1787828774.759732:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828774.759735:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787828774.759736:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1787828774.759738:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1787828774.759740:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787828774.759742:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828774.759744:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787828774.759746:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828774.759749:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787828774.759753:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d252c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646763520:192.168.203.143@tcp:38:kworker.0 00000100:00000001:2.0:1787828774.759757:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787828774.759759:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1787828774.759761:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828774.759763:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1787828774.759766:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828774.759774:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787828774.759775:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601341120 : -126234108210496 : ffff8d30d4d252c0) 00000100:00000001:2.0:1787828774.759780:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1787828774.759797:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:2.0:1787828774.759803:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1787828774.759805:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828774.759807:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828774.759809:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1787828774.759812:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828774.759817:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298247703 00000100:00000001:2.0:1787828774.759820:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298248960 00000100:00000001:2.0:1787828774.759823:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298247703 00000100:00000001:2.0:1787828774.759825:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298248960 00000100:00000001:2.0:1787828774.759828:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787828774.759830:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828774.759832:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1787828774.759836:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828774/real 1787828774] req@ffff8d30d4d252c0 x1874670646763520/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828799 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1787828774.759845:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787828774.759846:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1787828774.759849:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1787828774.759851:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787828774.759852:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828774.759854:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828774.759855:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787828774.759859:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d252c0 x1874670646763520/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828799 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787828774.759866:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828774.759869:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787828774.759871:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828774.759872:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828774.759874:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828774.759875:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787828774.759877:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787828774.759880:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828774.759900:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:2.0:1787828774.759902:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1787828774.759905:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d252c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646763520:192.168.203.143@tcp:38:kworker.0 00000100:00000001:2.0:1787828774.759911:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1787828774.759913:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787828774.759914:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828774.759916:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787828774.759918:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828774.759920:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787828774.759922:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d25680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646763648:192.168.203.143@tcp:38:kworker.0 00000100:00000001:2.0:1787828774.759925:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787828774.759926:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1787828774.759928:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828774.759929:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1787828774.759932:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828774.759937:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787828774.759939:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601342080 : -126234108209536 : ffff8d30d4d25680) 00000100:00000001:2.0:1787828774.759943:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1787828774.759973:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:2.0:1787828774.759978:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1787828774.759981:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828774.759983:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828774.759985:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1787828774.759988:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828774.759992:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298247703 00000100:00000001:2.0:1787828774.759995:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298248960 00000100:00000001:2.0:1787828774.759998:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298247703 00000100:00000001:2.0:1787828774.760001:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298248960 00000100:00000001:2.0:1787828774.760003:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787828774.760006:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828774.760009:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1787828774.760014:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828774/real 1787828774] req@ffff8d30d4d25680 x1874670646763648/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828799 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1787828774.760024:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787828774.760026:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1787828774.760028:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1787828774.760030:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787828774.760031:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828774.760032:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828774.760033:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787828774.760037:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d25680 x1874670646763648/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828799 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787828774.760042:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828774.760044:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787828774.760046:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828774.760047:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828774.760048:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828774.760049:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787828774.760050:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787828774.760052:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828774.760054:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:2.0:1787828774.760056:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1787828774.760058:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d25680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646763648:192.168.203.143@tcp:38:kworker.0 00000100:00000001:2.0:1787828774.760061:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1787828774.760062:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787828774.760063:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828774.760064:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787828774.760066:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828774.760068:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787828774.760070:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d26940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646763776:192.168.203.143@tcp:8:kworker.0 00000100:00000001:2.0:1787828774.760072:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787828774.760074:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1787828774.760075:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828774.760077:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1787828774.760079:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828774.760084:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787828774.760085:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601346880 : -126234108204736 : ffff8d30d4d26940) 00000100:00000001:2.0:1787828774.760088:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1787828774.760098:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:2.0:1787828774.760101:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1787828774.760103:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828774.760104:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828774.760105:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1787828774.760107:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828774.760110:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298247716 00000100:00000001:2.0:1787828774.760113:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298248960 00000100:00000001:2.0:1787828774.760115:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298247716 00000100:00000001:2.0:1787828774.760118:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298248960 00000100:00000001:2.0:1787828774.760121:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787828774.760123:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828774.760125:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1787828774.760130:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828774/real 1787828774] req@ffff8d30d4d26940 x1874670646763776/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828799 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1787828774.760139:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787828774.760141:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1787828774.760144:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1787828774.760146:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787828774.760148:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828774.760150:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828774.760152:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787828774.760157:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d26940 x1874670646763776/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828799 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787828774.760166:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828774.760169:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787828774.760173:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828774.760175:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828774.760176:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828774.760177:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787828774.760179:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787828774.760181:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828774.760184:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:2.0:1787828774.760185:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1787828774.760188:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d26940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646763776:192.168.203.143@tcp:8:kworker.0 00000100:00000001:2.0:1787828774.760191:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828774.760192:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828774.760194:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787828774.760195:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1787828774.760197:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787828774.760198:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:2.0:1787828774.760199:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:2.0:1787828774.760201:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787828774.760202:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1787828774.760205:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787828774.760206:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828774.760207:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828774.760208:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787828774.760209:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1787828774.760211:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787828774.760211:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:2.0:1787828774.760213:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:2.0:1787828774.760215:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787828774.760217:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1787828774.760219:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787828774.760221:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828774.760223:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828774.760224:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787828774.760225:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1787828774.760227:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787828774.760227:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:2.0:1787828774.760228:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:2.0:1787828774.760229:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787828774.760231:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1787828774.760234:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787828774.760235:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828774.760236:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828774.760239:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1787828774.760241:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828774.760244:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787828774.760246:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828775.654798:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828775.654806:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828775.654889:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828775.654893:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828775.654897:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828776.678718:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828776.678740:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828776.678808:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828776.678812:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828776.678815:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828777.702708:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828777.702714:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828777.702774:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828777.702779:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828777.702783:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828778.726715:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828778.726721:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828778.726769:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828778.726772:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828778.726775:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828779.750755:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828779.750763:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828779.750827:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828779.750832:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828779.750837:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787828779.878722:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828779.878742:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1787828779.878745:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787828779.878750:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787828779.878755:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787828779.878763:0:468:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 3580 00000100:00080000:0.0:1787828779.878776:0:468:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787828779.878780:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787828779.878782:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787828779.878787:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787828779.878789:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787828779.878793:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:0.0:1787828779.878795:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:0.0:1787828779.878798:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787828779.878799:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:0.0:1787828779.878805:0:468:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787828779.878807:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828779.878810:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787828779.878815:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787828779.878818:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828779.878820:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1787828779.878822:0:468:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828779.878837:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787828779.878839:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828779.878843:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787828779.878846:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787828779.878852:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828779.878868:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828779.878879:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d26940 x1874670646763904/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787828779.878890:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787828779.878895:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828779.878912:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787828779.878915:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828779.878918:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1787828779.878920:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787828779.878921:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787828779.878924:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787828779.878926:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3580 00000100:00080000:0.0:1787828779.878929:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787828779.878931:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787828779.878932:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787828779.878934:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787828779.878935:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787828779.878938:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000020:00000001:0.0:1787828779.878940:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:0.0:1787828779.878942:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787828779.878944:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:0.0:1787828779.878948:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787828779.878950:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828779.878952:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787828779.878958:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787828779.878960:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828779.878962:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1787828779.878968:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1787828779.878971:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828779.878974:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787828779.878982:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787828779.878983:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828779.878986:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1787828779.879006:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 02000000:00000001:0.0:1787828779.879008:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:2.0:1787828779.879009:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828779.879012:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828779.879016:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:0.0:1787828779.879017:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787828779.879020:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787828779.879022:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787828779.879024:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828779.879027:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787828779.879032:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d26940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646763904:192.168.203.143@tcp:250:kworker.0 00000100:00000001:0.0:1787828779.879038:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828779.879038:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787828779.879044:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1787828779.879049:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828779.879050:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d25680 x1874670646764032/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:2.0:1787828779.879053:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1787828779.879076:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828779.879081:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787828779.879084:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828779.879088:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787828779.879091:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828779.879095:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1787828779.879097:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787828779.879099:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787828779.879099:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787828779.879102:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:2.0:1787828779.879103:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601346880 : -126234108204736 : ffff8d30d4d26940) 00000100:00080000:0.0:1787828779.879108:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3580 00000100:00000001:2.0:1787828779.879111:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00080000:0.0:1787828779.879115:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787828779.879119:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787828779.879120:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787828779.879123:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787828779.879125:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787828779.879128:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000020:00000001:0.0:1787828779.879130:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:0.0:1787828779.879133:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787828779.879134:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:0.0:1787828779.879137:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787828779.879139:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828779.879140:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787828779.879144:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787828779.879146:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828779.879148:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1787828779.879150:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1787828779.879152:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828779.879153:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828779.879159:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000800:00000100:2.0:1787828779.879160:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 02000000:00000001:0.0:1787828779.879161:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828779.879166:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787828779.879167:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787828779.879169:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828779.879175:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828779.879180:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00100000:0.0:1787828779.879181:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d252c0 x1874670646764160/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787828779.879187:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828779.879192:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828779.879193:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787828779.879195:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787828779.879199:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828779.879204:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828779.879204:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787828779.879207:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:2.0:1787828779.879211:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4298254080 00000100:00080000:0.0:1787828779.879212:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1787828779.879216:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787828779.879219:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787828779.879221:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787828779.879224:0:468:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3580 00000100:00080000:0.0:1787828779.879228:0:468:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787828779.879229:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1787828779.879236:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787828779.879238:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787828779.879240:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787828779.879242:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:2.0:1787828779.879243:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4298254080 00000020:00000001:0.0:1787828779.879244:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000020:00000001:0.0:1787828779.879250:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:2.0:1787828779.879252:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1787828779.879253:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787828779.879257:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:2.0:1787828779.879259:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00080000:0.0:1787828779.879263:0:468:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787828779.879265:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828779.879267:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828779.879270:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 02000000:00000001:0.0:1787828779.879271:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787828779.879292:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787828779.879296:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787828779.879297:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828779/real 1787828779] req@ffff8d30d4d26940 x1874670646763904/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828804 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787828779.879298:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1787828779.879302:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1787828779.879309:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828779.879315:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:2.0:1787828779.879319:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787828779.879322:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 02000000:00000001:0.0:1787828779.879324:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:2.0:1787828779.879326:0:2390:0:(events.c:78:reply_in_callback()) Process entered 02000000:00000001:0.0:1787828779.879328:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828779.879330:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787828779.879333:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:2.0:1787828779.879333:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:0.0:1787828779.879339:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:2.0:1787828779.879339:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828779.879343:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:0.0:1787828779.879345:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828779.879353:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787828779.879357:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d26940 x1874670646763904/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828804 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828779.879363:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d27c00 x1874670646764288/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787828779.879375:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828779.879382:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787828779.879383:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787828779.879385:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828779.879388:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828779.879390:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828779.879391:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828779.879394:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828779.879397:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787828779.879399:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787828779.879401:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828779.879406:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:2.0:1787828779.879407:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1787828779.879411:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d26940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646763904:192.168.203.143@tcp:250:kworker.0 00000100:00000001:2.0:1787828779.879416:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828779.879418:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828779.879420:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787828779.879423:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1787828779.879444:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787828779.879446:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:2.0:1787828779.879448:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:2.0:1787828779.879453:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787828779.879455:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1787828779.879459:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787828779.879460:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828779.879462:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828779.879471:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1787828779.879473:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828779.879475:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787828779.879476:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1787828779.879478:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1787828779.879479:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787828779.879481:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828779.879483:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787828779.879485:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828779.879487:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787828779.879490:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d25680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646764032:192.168.203.143@tcp:38:kworker.0 00000100:00000001:2.0:1787828779.879493:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787828779.879495:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1787828779.879496:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828779.879498:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1787828779.879500:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828779.879507:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787828779.879508:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601342080 : -126234108209536 : ffff8d30d4d25680) 00000100:00000001:2.0:1787828779.879513:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1787828779.879525:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:2.0:1787828779.879530:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1787828779.879532:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828779.879533:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828779.879535:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1787828779.879537:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828779.879541:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298252822 00000100:00000001:2.0:1787828779.879544:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298254080 00000100:00000001:2.0:1787828779.879546:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298252822 00000100:00000001:2.0:1787828779.879548:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298254080 00000100:00000001:2.0:1787828779.879550:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787828779.879552:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828779.879554:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1787828779.879558:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828779/real 1787828779] req@ffff8d30d4d25680 x1874670646764032/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828804 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1787828779.879566:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787828779.879567:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1787828779.879569:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1787828779.879570:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787828779.879572:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828779.879573:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828779.879574:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787828779.879578:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d25680 x1874670646764032/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828804 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787828779.879584:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828779.879586:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787828779.879588:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828779.879590:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828779.879591:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828779.879592:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787828779.879594:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787828779.879597:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828779.879601:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:2.0:1787828779.879604:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1787828779.879608:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d25680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646764032:192.168.203.143@tcp:38:kworker.0 00000100:00000001:2.0:1787828779.879613:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1787828779.879666:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787828779.879668:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828779.879672:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787828779.879675:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828779.879678:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787828779.879683:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d252c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646764160:192.168.203.143@tcp:38:kworker.0 00000100:00000001:2.0:1787828779.879688:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787828779.879691:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1787828779.879693:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828779.879696:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1787828779.879700:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828779.879709:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787828779.879711:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601341120 : -126234108210496 : ffff8d30d4d252c0) 00000100:00000001:2.0:1787828779.879716:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1787828779.879732:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:2.0:1787828779.879737:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1787828779.879740:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828779.879741:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828779.879743:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1787828779.879746:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828779.879749:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298252823 00000100:00000001:2.0:1787828779.879751:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298254080 00000100:00000001:2.0:1787828779.879753:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298252823 00000100:00000001:2.0:1787828779.879755:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298254080 00000100:00000001:2.0:1787828779.879757:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787828779.879759:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828779.879761:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1787828779.879764:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828779/real 1787828779] req@ffff8d30d4d252c0 x1874670646764160/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828804 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1787828779.879771:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787828779.879772:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1787828779.879774:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1787828779.879776:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787828779.879777:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828779.879778:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828779.879780:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787828779.879784:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d252c0 x1874670646764160/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828804 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787828779.879790:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828779.879792:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787828779.879794:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828779.879796:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828779.879805:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828779.879807:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787828779.879808:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787828779.879810:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828779.879812:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:2.0:1787828779.879814:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1787828779.879817:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d252c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646764160:192.168.203.143@tcp:38:kworker.0 00000100:00000001:2.0:1787828779.879820:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1787828779.879821:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787828779.879823:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828779.879825:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787828779.879826:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828779.879828:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787828779.879831:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d27c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646764288:192.168.203.143@tcp:8:kworker.0 00000100:00000001:2.0:1787828779.879833:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787828779.879835:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1787828779.879836:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828779.879838:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1787828779.879840:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828779.879845:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787828779.879846:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601351680 : -126234108199936 : ffff8d30d4d27c00) 00000100:00000001:2.0:1787828779.879849:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1787828779.879858:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:2.0:1787828779.879861:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1787828779.879863:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828779.879865:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828779.879867:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1787828779.879869:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828779.879871:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298252836 00000100:00000001:2.0:1787828779.879873:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298254080 00000100:00000001:2.0:1787828779.879875:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298252836 00000100:00000001:2.0:1787828779.879878:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298254080 00000100:00000001:2.0:1787828779.879879:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787828779.879881:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828779.879882:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1787828779.879885:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828779/real 1787828779] req@ffff8d30d4d27c00 x1874670646764288/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828804 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1787828779.879891:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787828779.879892:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1787828779.879895:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1787828779.879896:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787828779.879898:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828779.879900:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828779.879902:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787828779.879907:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d27c00 x1874670646764288/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828804 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787828779.879916:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828779.879919:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787828779.879922:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828779.879924:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828779.879926:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828779.879929:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787828779.879932:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787828779.879935:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828779.879940:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:2.0:1787828779.879943:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1787828779.879964:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d27c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646764288:192.168.203.143@tcp:8:kworker.0 00000100:00000001:2.0:1787828779.879969:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828779.879972:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828779.879974:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787828779.879976:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1787828779.879978:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787828779.879980:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:2.0:1787828779.879982:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:2.0:1787828779.879984:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787828779.879985:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1787828779.879989:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787828779.879990:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828779.879991:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828779.879992:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787828779.879993:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1787828779.879994:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787828779.879995:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:2.0:1787828779.879997:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:2.0:1787828779.879998:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787828779.879999:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1787828779.880000:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787828779.880001:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828779.880002:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828779.880003:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787828779.880004:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1787828779.880005:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787828779.880005:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:2.0:1787828779.880006:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:2.0:1787828779.880007:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787828779.880008:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1787828779.880010:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787828779.880011:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828779.880012:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828779.880031:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1787828779.880033:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828779.880034:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787828779.880036:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828780.774718:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828780.774724:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828780.774777:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828780.774780:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828780.774783:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828781.798732:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828781.798741:0:815:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1787828781.798744:0:815:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1787828781.798747:0:815:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828781.798752:0:815:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1787828781.798755:0:815:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828781.798758:0:815:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828781.798761:0:815:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828781.798831:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828781.798836:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828781.798839:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828782.822710:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828782.822716:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828782.822770:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828782.822774:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828782.822776:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828783.846686:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828783.846691:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828783.846747:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828783.846751:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828783.846754:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828784.870729:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828784.870737:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828784.870819:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828784.870823:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828784.870826:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787828784.998714:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828784.998721:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1787828784.998723:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787828784.998727:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787828784.998731:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787828784.998737:0:40798:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 3585 00000100:00080000:0.0:1787828784.998747:0:40798:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787828784.998749:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787828784.998751:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787828784.998754:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787828784.998756:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787828784.998758:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:0.0:1787828784.998760:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:0.0:1787828784.998761:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787828784.998762:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:0.0:1787828784.998765:0:40798:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787828784.998767:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828784.998769:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787828784.998772:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787828784.998774:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828784.998775:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1787828784.998777:0:40798:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828784.998787:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787828784.998788:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828784.998790:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787828784.998792:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787828784.998796:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828784.998805:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828784.998812:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d270c0 x1874670646764416/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787828784.998820:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787828784.998823:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828784.998837:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787828784.998840:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828784.998842:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1787828784.998843:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787828784.998845:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787828784.998847:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787828784.998849:0:40798:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3585 00000100:00080000:0.0:1787828784.998851:0:40798:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787828784.998852:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787828784.998853:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787828784.998855:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787828784.998856:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787828784.998858:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000020:00000001:0.0:1787828784.998859:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:0.0:1787828784.998861:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787828784.998862:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:0.0:1787828784.998866:0:40798:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787828784.998868:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828784.998871:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787828784.998876:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787828784.998877:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828784.998879:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1787828784.998883:0:40798:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00000001:2.0:1787828784.998883:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000008:00000001:0.0:1787828784.998892:0:40798:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828784.998895:0:40798:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828784.998895:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1787828784.998903:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 02000000:00000001:0.0:1787828784.998904:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787828784.998907:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828784.998908:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787828784.998915:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828784.998917:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787828784.998924:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787828784.998928:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828784.998928:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787828784.998938:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787828784.998940:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828784.998948:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828784.998954:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d24000 x1874670646764544/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787828784.998962:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d270c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646764416:192.168.203.143@tcp:250:kworker.0 00000100:00000001:0.0:1787828784.998970:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787828784.998973:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:0.0:1787828784.998974:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828784.998978:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828784.998980:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000100:00080000:0.0:1787828784.998982:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 02000000:00000001:2.0:1787828784.998984:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787828784.998988:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1787828784.998991:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787828784.998992:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 02000000:00000001:2.0:1787828784.998992:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:0.0:1787828784.998999:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787828784.999002:0:40798:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3585 00000100:00080000:0.0:1787828784.999003:0:40798:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787828784.999005:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787828784.999006:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 02000000:00000001:2.0:1787828784.999007:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787828784.999016:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787828784.999018:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787828784.999020:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000020:00000001:0.0:1787828784.999021:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:0.0:1787828784.999023:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787828784.999024:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:0.0:1787828784.999027:0:40798:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787828784.999028:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828784.999029:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787828784.999031:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787828784.999033:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828784.999034:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1787828784.999035:0:40798:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1787828784.999037:0:40798:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828784.999038:0:40798:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828784.999040:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 02000000:00000001:0.0:1787828784.999041:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787828784.999049:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828784.999049:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601348800 : -126234108202816 : ffff8d30d4d270c0) 00000100:00000001:0.0:1787828784.999056:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787828784.999063:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:2.0:1787828784.999065:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 02000000:00000001:0.0:1787828784.999066:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828784.999074:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828784.999078:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d24f00 x1874670646764672/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787828784.999087:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787828784.999088:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828784.999090:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787828784.999092:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828784.999094:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1787828784.999095:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787828784.999096:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787828784.999097:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787828784.999100:0:40798:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3585 00000100:00080000:0.0:1787828784.999105:0:40798:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787828784.999107:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787828784.999108:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787828784.999109:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787828784.999110:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787828784.999112:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000020:00000001:0.0:1787828784.999113:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:0.0:1787828784.999114:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787828784.999115:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000800:00000100:2.0:1787828784.999120:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00080000:0.0:1787828784.999121:0:40798:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787828784.999130:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828784.999133:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787828784.999135:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787828784.999138:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828784.999140:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1787828784.999144:0:40798:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00000001:2.0:1787828784.999145:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000008:00000001:0.0:1787828784.999147:0:40798:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828784.999150:0:40798:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828784.999152:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 02000000:00000001:0.0:1787828784.999153:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787828784.999156:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828784.999156:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828784.999176:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787828784.999177:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787828784.999186:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787828784.999190:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828784.999192:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828784.999194:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828784.999198:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4298259200 00000100:00100000:0.0:1787828784.999207:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d25a40 x1874670646764800/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787828784.999210:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1787828784.999221:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787828784.999224:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828784.999227:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828784.999227:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4298259200 00000100:00000001:2.0:1787828784.999235:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 00000100:00000001:2.0:1787828784.999242:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787828784.999246:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828784.999249:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1787828784.999258:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828784/real 1787828784] req@ffff8d30d4d270c0 x1874670646764416/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828809 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1787828784.999268:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787828784.999270:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1787828784.999273:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1787828784.999276:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787828784.999277:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828784.999280:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828784.999282:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787828784.999285:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d270c0 x1874670646764416/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828809 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787828784.999291:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828784.999294:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787828784.999296:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828784.999298:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828784.999298:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828784.999301:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787828784.999302:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787828784.999305:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828784.999310:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:2.0:1787828784.999312:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1787828784.999316:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d270c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646764416:192.168.203.143@tcp:250:kworker.0 00000100:00000001:2.0:1787828784.999319:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828784.999321:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828784.999323:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787828784.999326:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1787828784.999329:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787828784.999331:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:2.0:1787828784.999333:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:2.0:1787828784.999337:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787828784.999339:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1787828784.999343:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787828784.999344:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828784.999345:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828784.999353:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1787828784.999354:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828784.999356:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787828784.999358:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1787828784.999359:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1787828784.999360:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787828784.999361:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828784.999363:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787828784.999365:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828784.999367:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787828784.999369:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d24000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646764544:192.168.203.143@tcp:38:kworker.0 00000100:00000001:2.0:1787828784.999372:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787828784.999373:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1787828784.999375:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828784.999376:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1787828784.999378:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828784.999383:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787828784.999384:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601336320 : -126234108215296 : ffff8d30d4d24000) 00000100:00000001:2.0:1787828784.999388:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1787828784.999397:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:2.0:1787828784.999400:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1787828784.999402:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828784.999403:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828784.999404:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1787828784.999406:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828784.999410:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298257942 00000100:00000001:2.0:1787828784.999412:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298259200 00000100:00000001:2.0:1787828784.999414:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298257942 00000100:00000001:2.0:1787828784.999416:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298259200 00000100:00000001:2.0:1787828784.999418:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787828784.999420:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828784.999421:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1787828784.999424:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828784/real 1787828784] req@ffff8d30d4d24000 x1874670646764544/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828809 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1787828784.999430:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787828784.999431:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1787828784.999432:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1787828784.999434:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787828784.999435:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828784.999436:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828784.999437:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787828784.999439:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d24000 x1874670646764544/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828809 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787828784.999445:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828784.999446:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787828784.999448:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828784.999449:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828784.999450:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828784.999451:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787828784.999453:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787828784.999454:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828784.999456:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:2.0:1787828784.999458:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1787828784.999460:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d24000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646764544:192.168.203.143@tcp:38:kworker.0 00000100:00000001:2.0:1787828784.999462:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1787828784.999464:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787828784.999465:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828784.999466:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787828784.999468:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828784.999470:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787828784.999472:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d24f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646764672:192.168.203.143@tcp:38:kworker.0 00000100:00000001:2.0:1787828784.999475:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787828784.999476:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1787828784.999477:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828784.999478:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1787828784.999480:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828784.999483:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787828784.999484:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601340160 : -126234108211456 : ffff8d30d4d24f00) 00000100:00000001:2.0:1787828784.999486:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1787828784.999492:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:2.0:1787828784.999495:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1787828784.999496:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828784.999497:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828784.999498:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1787828784.999500:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828784.999503:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298257942 00000100:00000001:2.0:1787828784.999505:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298259200 00000100:00000001:2.0:1787828784.999506:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298257942 00000100:00000001:2.0:1787828784.999508:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298259200 00000100:00000001:2.0:1787828784.999509:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787828784.999511:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828784.999512:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1787828784.999514:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828784/real 1787828784] req@ffff8d30d4d24f00 x1874670646764672/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828809 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1787828784.999519:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787828784.999520:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1787828784.999521:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1787828784.999522:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787828784.999523:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828784.999524:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828784.999525:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787828784.999528:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d24f00 x1874670646764672/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828809 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787828784.999533:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828784.999535:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787828784.999537:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828784.999538:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828784.999538:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828784.999539:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787828784.999540:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787828784.999542:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828784.999544:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:2.0:1787828784.999545:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1787828784.999547:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d24f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646764672:192.168.203.143@tcp:38:kworker.0 00000100:00000001:2.0:1787828784.999549:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1787828784.999550:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787828784.999551:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828784.999553:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787828784.999554:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828784.999555:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787828784.999557:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d25a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646764800:192.168.203.143@tcp:8:kworker.0 00000100:00000001:2.0:1787828784.999559:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787828784.999560:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1787828784.999561:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828784.999562:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1787828784.999564:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828784.999568:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787828784.999569:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601343040 : -126234108208576 : ffff8d30d4d25a40) 00000100:00000001:2.0:1787828784.999572:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1787828784.999577:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:2.0:1787828784.999580:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1787828784.999581:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828784.999583:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828784.999584:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1787828784.999586:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828784.999589:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298257955 00000100:00000001:2.0:1787828784.999590:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298259200 00000100:00000001:2.0:1787828784.999637:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298257956 00000100:00000001:2.0:1787828784.999640:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298259200 00000100:00000001:2.0:1787828784.999642:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787828784.999643:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828784.999645:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1787828784.999648:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828784/real 1787828784] req@ffff8d30d4d25a40 x1874670646764800/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828809 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1787828784.999654:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787828784.999655:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1787828784.999658:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1787828784.999659:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787828784.999660:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828784.999662:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828784.999663:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787828784.999666:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d25a40 x1874670646764800/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828809 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787828784.999671:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828784.999673:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787828784.999674:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828784.999676:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828784.999676:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828784.999678:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787828784.999679:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787828784.999680:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828784.999682:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:2.0:1787828784.999684:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1787828784.999687:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d25a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646764800:192.168.203.143@tcp:8:kworker.0 00000100:00000001:2.0:1787828784.999689:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828784.999691:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828784.999692:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787828784.999693:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1787828784.999695:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787828784.999696:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:2.0:1787828784.999697:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:2.0:1787828784.999698:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787828784.999699:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1787828784.999702:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787828784.999703:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828784.999704:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828784.999705:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787828784.999706:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1787828784.999707:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787828784.999708:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:2.0:1787828784.999709:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:2.0:1787828784.999710:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787828784.999711:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1787828784.999712:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787828784.999713:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828784.999714:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828784.999715:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787828784.999715:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1787828784.999716:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787828784.999717:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:2.0:1787828784.999718:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:2.0:1787828784.999719:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787828784.999719:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1787828784.999720:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787828784.999721:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828784.999722:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828784.999724:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1787828784.999725:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828784.999727:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787828784.999728:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828785.894681:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828785.894687:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828785.894756:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828785.894761:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828785.894765:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828786.918689:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828786.918694:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828786.918804:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828786.918809:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828786.918813:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828787.942679:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828787.942686:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828787.942738:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828787.942741:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828787.942744:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828788.966675:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828788.966681:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828788.966722:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828788.966726:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828788.966729:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828789.990691:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828789.990697:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828789.990765:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828789.990770:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828789.990773:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787828790.118748:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828790.118758:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1787828790.118761:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787828790.118766:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787828790.118773:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787828790.118781:0:40798:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 3590 00000100:00080000:0.0:1787828790.118792:0:40798:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787828790.118797:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787828790.118799:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787828790.118804:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787828790.118806:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787828790.118809:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:0.0:1787828790.118813:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:0.0:1787828790.118816:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787828790.118819:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:0.0:1787828790.118825:0:40798:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787828790.118829:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828790.118832:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787828790.118835:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787828790.118838:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828790.118842:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1787828790.118845:0:40798:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828790.118854:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787828790.118857:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828790.118861:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787828790.118864:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787828790.118869:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828790.118882:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828790.118892:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d25a40 x1874670646764928/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787828790.118904:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787828790.118909:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828790.118926:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787828790.118933:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828790.118940:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1787828790.118944:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787828790.118947:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787828790.118952:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787828790.118957:0:40798:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3590 00000100:00080000:0.0:1787828790.118978:0:40798:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787828790.118981:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787828790.118982:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787828790.118985:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787828790.118988:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787828790.118992:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000020:00000001:0.0:1787828790.118997:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:1.0:1787828790.118997:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787828790.119005:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787828790.119008:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:1.0:1787828790.119008:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1787828790.119016:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00080000:0.0:1787828790.119022:0:40798:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:1.0:1787828790.119023:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787828790.119040:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828790.119045:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828790.119050:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1787828790.119051:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787828790.119059:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787828790.119060:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787828790.119071:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828790.119071:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828790.119080:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00100000:1.0:1787828790.119084:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d25a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646764928:192.168.203.143@tcp:250:kworker.0 00000008:00100000:0.0:1787828790.119087:0:40798:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00000001:1.0:1787828790.119090:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000008:00000001:0.0:1787828790.119095:0:40798:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828790.119098:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:0.0:1787828790.119104:0:40798:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828790.119105:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828790.119113:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1787828790.119114:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787828790.119119:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828790.119121:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828790.119122:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787828790.119126:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787828790.119130:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828790.119139:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828790.119149:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d24f00 x1874670646765056/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787828790.119149:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787828790.119155:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601343040 : -126234108208576 : ffff8d30d4d25a40) 00000100:00000001:1.0:1787828790.119165:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:0.0:1787828790.119171:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787828790.119175:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828790.119181:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787828790.119184:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828790.119189:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1787828790.119191:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787828790.119192:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787828790.119196:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787828790.119200:0:40798:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3590 00000100:00080000:0.0:1787828790.119206:0:40798:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787828790.119210:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787828790.119216:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000800:00000100:1.0:1787828790.119216:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000020:00000001:0.0:1787828790.119224:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787828790.119226:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787828790.119229:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:1.0:1787828790.119232:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000020:00000001:0.0:1787828790.119234:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:0.0:1787828790.119239:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787828790.119239:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787828790.119243:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828790.119245:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:1.0:1787828790.119247:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787828790.119256:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787828790.119260:0:40798:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:1.0:1787828790.119263:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4298264320 00000100:00000001:0.0:1787828790.119271:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828790.119276:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1787828790.119278:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 02000000:00000001:0.0:1787828790.119281:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787828790.119288:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4298264320 02000000:00000001:0.0:1787828790.119289:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828790.119299:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:1.0:1787828790.119301:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 00000008:00100000:0.0:1787828790.119304:0:40798:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00000001:1.0:1787828790.119309:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000008:00000001:0.0:1787828790.119314:0:40798:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828790.119315:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828790.119322:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00000001:0.0:1787828790.119324:0:40798:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828790.119334:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00100000:1.0:1787828790.119334:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828790/real 1787828790] req@ffff8d30d4d25a40 x1874670646764928/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828815 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:0.0:1787828790.119341:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828790.119352:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000400:00000001:1.0:1787828790.119354:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 02000000:00000001:0.0:1787828790.119357:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000400:00000001:1.0:1787828790.119357:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 02000000:00000001:0.0:1787828790.119365:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828790.119366:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787828790.119372:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787828790.119375:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828790.119378:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828790.119378:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787828790.119382:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787828790.119393:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d25a40 x1874670646764928/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828815 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787828790.119396:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d24000 x1874670646765184/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787828790.119411:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828790.119418:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787828790.119429:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787828790.119431:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828790.119436:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828790.119437:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787828790.119440:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828790.119445:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828790.119447:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787828790.119460:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787828790.119468:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828790.119470:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828790.119480:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787828790.119484:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787828790.119485:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:1.0:1787828790.119489:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00080000:0.0:1787828790.119491:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1787828790.119500:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d25a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646764928:192.168.203.143@tcp:250:kworker.0 00000100:00000001:0.0:1787828790.119502:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:1.0:1787828790.119516:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:0.0:1787828790.119518:0:40798:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3590 00000100:00000001:1.0:1787828790.119520:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787828790.119523:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787828790.119529:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 00000100:00080000:0.0:1787828790.119531:0:40798:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 02000000:00000001:1.0:1787828790.119534:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00000001:0.0:1787828790.119540:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000020:00000001:1.0:1787828790.119541:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787828790.119545:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 00000100:00000001:0.0:1787828790.119546:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 02000000:00000001:1.0:1787828790.119551:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787828790.119556:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000020:00000001:0.0:1787828790.119574:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787828790.119579:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:1.0:1787828790.119580:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787828790.119585:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1787828790.119588:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:1.0:1787828790.119590:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1787828790.119650:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:1.0:1787828790.119652:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787828790.119656:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828790.119661:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787828790.119662:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787828790.119669:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787828790.119670:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:1.0:1787828790.119672:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787828790.119677:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787828790.119680:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787828790.119684:0:40798:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 02000000:00000001:1.0:1787828790.119686:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787828790.119691:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787828790.119692:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828790.119695:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828790.119699:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00100000:1.0:1787828790.119701:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d24f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646765056:192.168.203.143@tcp:38:kworker.0 02000000:00000001:0.0:1787828790.119705:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787828790.119707:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787828790.119710:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787828790.119714:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828790.119716:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828790.119717:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:0.0:1787828790.119724:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 02000000:00000001:1.0:1787828790.119724:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000008:00100000:0.0:1787828790.119734:0:40798:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00000001:1.0:1787828790.119737:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787828790.119741:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601340160 : -126234108211456 : ffff8d30d4d24f00) 00000008:00000001:0.0:1787828790.119743:0:40798:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828790.119747:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:0.0:1787828790.119750:0:40798:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828790.119759:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787828790.119763:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:1.0:1787828790.119767:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:0.0:1787828790.119769:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787828790.119773:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:1.0:1787828790.119774:0:2390:0:(events.c:34:request_out_callback()) Process entered 02000000:00000001:0.0:1787828790.119777:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828790.119779:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787828790.119782:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828790.119784:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787828790.119787:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828790.119789:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828790.119792:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298263063 00000100:00000001:1.0:1787828790.119796:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298264320 00000100:00000001:1.0:1787828790.119802:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298263063 00000100:00100000:0.0:1787828790.119804:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d270c0 x1874670646765312/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787828790.119806:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298264320 00000100:00000001:1.0:1787828790.119809:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787828790.119819:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828790.119826:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787828790.119834:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828790/real 1787828790] req@ffff8d30d4d24f00 x1874670646765056/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828815 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787828790.119844:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000400:00000001:1.0:1787828790.119850:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000001:0.0:1787828790.119853:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:1.0:1787828790.119853:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787828790.119857:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787828790.119860:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787828790.119863:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828790.119866:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828790.119867:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828790.119868:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787828790.119880:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d24f00 x1874670646765056/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828815 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787828790.119887:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828790.119889:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787828790.119892:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828790.119893:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787828790.119894:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828790.119896:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787828790.119898:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787828790.119901:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828790.119904:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787828790.119906:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787828790.119909:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d24f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646765056:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787828790.119913:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787828790.119914:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787828790.119916:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828790.119917:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787828790.119919:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787828790.119921:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787828790.119923:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d24000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646765184:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787828790.119926:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787828790.119927:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787828790.119929:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828790.119930:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787828790.119933:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828790.119937:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787828790.119938:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601336320 : -126234108215296 : ffff8d30d4d24000) 00000100:00000001:1.0:1787828790.119942:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1787828790.119950:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787828790.119953:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787828790.119955:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787828790.119956:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828790.119969:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787828790.119971:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828790.119974:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298263063 00000100:00000001:1.0:1787828790.119976:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298264320 00000100:00000001:1.0:1787828790.119978:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298263063 00000100:00000001:1.0:1787828790.119980:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298264320 00000100:00000001:1.0:1787828790.119982:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787828790.119984:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828790.119986:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787828790.119990:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828790/real 1787828790] req@ffff8d30d4d24000 x1874670646765184/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828815 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787828790.120000:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787828790.120002:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787828790.120005:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787828790.120007:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787828790.120008:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828790.120011:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787828790.120013:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787828790.120018:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d24000 x1874670646765184/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828815 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787828790.120037:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828790.120040:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787828790.120044:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828790.120046:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787828790.120047:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828790.120049:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787828790.120051:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787828790.120055:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828790.120059:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787828790.120061:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787828790.120064:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d24000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646765184:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787828790.120067:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828790.120069:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787828790.120070:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787828790.120072:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787828790.120074:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787828790.120075:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787828790.120077:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787828790.120079:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787828790.120080:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787828790.120084:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787828790.120085:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828790.120086:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787828790.120087:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787828790.120088:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787828790.120090:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787828790.120091:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787828790.120092:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787828790.120094:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787828790.120095:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787828790.120096:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787828790.120097:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828790.120098:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828790.120101:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787828790.120103:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828790.120106:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787828790.120107:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1787828790.120109:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787828790.120110:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787828790.120113:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828790.120116:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787828790.120118:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787828790.120121:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787828790.120125:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d270c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646765312:192.168.203.143@tcp:8:kworker.0 00000100:00000001:1.0:1787828790.120130:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787828790.120132:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787828790.120134:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828790.120137:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787828790.120141:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828790.120149:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787828790.120151:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601348800 : -126234108202816 : ffff8d30d4d270c0) 00000100:00000001:1.0:1787828790.120155:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1787828790.120169:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787828790.120173:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787828790.120175:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787828790.120176:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828790.120178:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787828790.120180:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828790.120183:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298263076 00000100:00000001:1.0:1787828790.120185:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298264320 00000100:00000001:1.0:1787828790.120188:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298263076 00000100:00000001:1.0:1787828790.120189:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298264320 00000100:00000001:1.0:1787828790.120191:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787828790.120193:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828790.120194:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787828790.120197:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828790/real 1787828790] req@ffff8d30d4d270c0 x1874670646765312/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828815 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787828790.120203:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787828790.120204:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787828790.120206:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787828790.120207:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787828790.120209:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828790.120211:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787828790.120212:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787828790.120215:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d270c0 x1874670646765312/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828815 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787828790.120220:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828790.120222:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787828790.120224:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828790.120225:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787828790.120226:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828790.120227:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787828790.120228:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787828790.120230:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828790.120232:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787828790.120234:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787828790.120236:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d270c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646765312:192.168.203.143@tcp:8:kworker.0 00000100:00000001:1.0:1787828790.120239:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828790.120240:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787828790.120242:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787828790.120243:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787828790.120245:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787828790.120246:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787828790.120247:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787828790.120249:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787828790.120250:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787828790.120253:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787828790.120254:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828790.120256:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828790.120258:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787828790.120260:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828790.120261:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787828790.120263:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828791.014708:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828791.014713:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828791.014764:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828791.014767:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828791.014770:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828792.038725:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828792.038735:0:815:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1787828792.038737:0:815:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1787828792.038740:0:815:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828792.038746:0:815:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1787828792.038748:0:815:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828792.038750:0:815:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828792.038752:0:815:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828792.038825:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828792.038831:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828792.038835:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828793.062734:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828793.062742:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828793.062848:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828793.062856:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828793.062860:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828794.086682:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828794.086687:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828794.086731:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828794.086751:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828794.086754:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828795.110761:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828795.110770:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828795.110870:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828795.110875:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828795.110879:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787828795.238678:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828795.238686:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1787828795.238688:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787828795.238691:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787828795.238694:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787828795.238700:0:40798:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 3595 00000100:00080000:0.0:1787828795.238709:0:40798:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787828795.238711:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787828795.238712:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787828795.238716:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787828795.238718:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787828795.238721:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:0.0:1787828795.238723:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:0.0:1787828795.238724:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787828795.238725:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:0.0:1787828795.238729:0:40798:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787828795.238730:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828795.238732:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787828795.238735:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787828795.238744:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828795.238746:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1787828795.238748:0:40798:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828795.238755:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787828795.238756:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828795.238759:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787828795.238761:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787828795.238764:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828795.238772:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828795.238780:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d27c00 x1874670646765440/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787828795.238788:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787828795.238791:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828795.238802:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787828795.238804:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828795.238806:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1787828795.238808:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787828795.238809:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787828795.238811:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787828795.238813:0:40798:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3595 00000100:00080000:0.0:1787828795.238816:0:40798:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787828795.238817:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787828795.238819:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787828795.238821:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787828795.238823:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787828795.238825:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000020:00000001:0.0:1787828795.238827:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:0.0:1787828795.238828:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787828795.238829:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:0.0:1787828795.238832:0:40798:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787828795.238835:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828795.238836:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:0.0:1787828795.238837:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787828795.238844:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787828795.238844:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 02000000:00000001:0.0:1787828795.238851:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828795.238851:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1787828795.238860:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:1.0:1787828795.238860:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000008:00100000:0.0:1787828795.238870:0:40798:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00000001:1.0:1787828795.238873:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:0.0:1787828795.238875:0:40798:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828795.238884:0:40798:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828795.238887:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787828795.238889:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787828795.238892:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828795.238892:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787828795.238909:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1787828795.238909:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828795.238919:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787828795.238925:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787828795.238925:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d27c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646765440:192.168.203.143@tcp:250:kworker.0 00000100:00000001:0.0:1787828795.238931:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828795.238939:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d252c0 x1874670646765568/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787828795.238940:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:0.0:1787828795.238952:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 02000000:00000001:1.0:1787828795.238953:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:0.0:1787828795.238958:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828795.238959:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828795.238969:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:0.0:1787828795.238976:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828795.238980:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787828795.238981:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828795.238986:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1787828795.238988:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787828795.238989:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787828795.238993:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787828795.238995:0:40798:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3595 00000100:00080000:0.0:1787828795.238998:0:40798:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787828795.239000:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787828795.239001:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:1.0:1787828795.239003:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000020:00000001:0.0:1787828795.239004:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:1.0:1787828795.239006:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601351680 : -126234108199936 : ffff8d30d4d27c00) 00000020:00000001:0.0:1787828795.239007:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787828795.239012:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:1.0:1787828795.239013:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000020:00000001:0.0:1787828795.239017:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:0.0:1787828795.239019:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787828795.239020:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:0.0:1787828795.239023:0:40798:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787828795.239024:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828795.239026:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787828795.239028:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787828795.239030:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828795.239032:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1787828795.239034:0:40798:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1787828795.239037:0:40798:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828795.239039:0:40798:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828795.239044:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787828795.239046:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828795.239048:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787828795.239049:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787828795.239052:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:1.0:1787828795.239060:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:0.0:1787828795.239061:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828795.239073:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d25680 x1874670646765696/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787828795.239077:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1787828795.239085:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787828795.239085:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828795.239091:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828795.239092:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828795.239094:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828795.239097:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00080000:0.0:1787828795.239098:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828795.239103:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1787828795.239105:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787828795.239107:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787828795.239110:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:1.0:1787828795.239111:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787828795.239114:0:40798:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3595 00000100:00080000:0.0:1787828795.239121:0:40798:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1787828795.239121:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4298269440 00000100:00000001:0.0:1787828795.239132:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787828795.239134:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787828795.239137:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:1.0:1787828795.239140:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 00000020:00000001:0.0:1787828795.239143:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787828795.239146:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000020:00000001:0.0:1787828795.239148:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:0.0:1787828795.239149:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787828795.239151:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:0.0:1787828795.239153:0:40798:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787828795.239155:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828795.239157:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1787828795.239157:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4298269440 02000000:00000001:0.0:1787828795.239165:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787828795.239165:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 02000000:00000001:0.0:1787828795.239175:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828795.239177:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787828795.239179:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:1.0:1787828795.239181:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000008:00100000:0.0:1787828795.239182:0:40798:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00000001:1.0:1787828795.239185:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000008:00000001:0.0:1787828795.239187:0:40798:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828795.239191:0:40798:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828795.239194:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00100000:1.0:1787828795.239196:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828795/real 1787828795] req@ffff8d30d4d27c00 x1874670646765440/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828820 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:0.0:1787828795.239197:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828795.239204:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787828795.239209:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787828795.239214:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828795.239223:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:1.0:1787828795.239223:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787828795.239227:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787828795.239232:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00100000:0.0:1787828795.239233:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d26940 x1874670646765824/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787828795.239235:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787828795.239240:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828795.239247:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787828795.239256:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828795.239257:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787828795.239265:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828795.239269:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787828795.239270:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d27c00 x1874670646765440/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828820 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787828795.239282:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828795.239287:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787828795.239289:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828795.239290:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787828795.239292:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828795.239294:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787828795.239297:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787828795.239300:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828795.239304:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787828795.239306:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787828795.239308:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d27c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646765440:192.168.203.143@tcp:250:kworker.0 00000100:00000001:1.0:1787828795.239312:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828795.239314:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787828795.239315:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787828795.239317:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787828795.239322:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787828795.239324:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787828795.239326:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787828795.239330:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787828795.239331:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787828795.239335:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787828795.239336:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828795.239338:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828795.239345:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787828795.239347:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828795.239349:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787828795.239351:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1787828795.239352:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787828795.239353:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787828795.239355:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828795.239357:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787828795.239359:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787828795.239361:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787828795.239363:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d252c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646765568:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787828795.239365:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787828795.239367:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787828795.239368:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828795.239369:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787828795.239371:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828795.239375:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787828795.239377:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601341120 : -126234108210496 : ffff8d30d4d252c0) 00000100:00000001:1.0:1787828795.239381:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1787828795.239391:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787828795.239394:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787828795.239396:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787828795.239397:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828795.239398:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787828795.239400:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828795.239404:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298268182 00000100:00000001:1.0:1787828795.239406:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298269440 00000100:00000001:1.0:1787828795.239408:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298268182 00000100:00000001:1.0:1787828795.239409:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298269440 00000100:00000001:1.0:1787828795.239411:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787828795.239412:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828795.239413:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787828795.239416:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828795/real 1787828795] req@ffff8d30d4d252c0 x1874670646765568/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828820 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787828795.239422:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787828795.239422:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787828795.239424:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787828795.239425:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787828795.239426:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828795.239427:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787828795.239428:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787828795.239431:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d252c0 x1874670646765568/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828820 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787828795.239436:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828795.239438:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787828795.239439:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828795.239440:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787828795.239441:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828795.239442:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787828795.239443:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787828795.239445:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828795.239447:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787828795.239448:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787828795.239450:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d252c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646765568:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787828795.239452:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787828795.239454:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787828795.239455:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828795.239457:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787828795.239458:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787828795.239459:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787828795.239461:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d25680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646765696:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787828795.239463:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787828795.239464:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787828795.239465:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828795.239466:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787828795.239468:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828795.239471:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787828795.239472:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601342080 : -126234108209536 : ffff8d30d4d25680) 00000100:00000001:1.0:1787828795.239474:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1787828795.239480:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787828795.239483:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787828795.239485:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787828795.239488:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828795.239490:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787828795.239493:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828795.239496:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298268182 00000100:00000001:1.0:1787828795.239498:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298269440 00000100:00000001:1.0:1787828795.239501:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298268182 00000100:00000001:1.0:1787828795.239503:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298269440 00000100:00000001:1.0:1787828795.239505:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787828795.239507:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828795.239509:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787828795.239514:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828795/real 1787828795] req@ffff8d30d4d25680 x1874670646765696/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828820 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787828795.239539:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787828795.239540:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787828795.239543:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787828795.239545:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787828795.239547:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828795.239548:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787828795.239549:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787828795.239553:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d25680 x1874670646765696/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828820 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787828795.239558:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828795.239560:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787828795.239561:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828795.239562:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787828795.239563:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828795.239564:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787828795.239565:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787828795.239567:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828795.239569:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787828795.239570:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787828795.239573:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d25680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646765696:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787828795.239576:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787828795.239577:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787828795.239578:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828795.239579:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787828795.239580:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787828795.239582:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787828795.239584:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d26940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646765824:192.168.203.143@tcp:8:kworker.0 00000100:00000001:1.0:1787828795.239586:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787828795.239588:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787828795.239589:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828795.239590:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787828795.239593:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828795.239597:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787828795.239598:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601346880 : -126234108204736 : ffff8d30d4d26940) 00000100:00000001:1.0:1787828795.239621:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1787828795.239630:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787828795.239634:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787828795.239635:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787828795.239637:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828795.239638:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787828795.239640:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828795.239642:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298268196 00000100:00000001:1.0:1787828795.239644:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298269440 00000100:00000001:1.0:1787828795.239646:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298268196 00000100:00000001:1.0:1787828795.239648:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298269440 00000100:00000001:1.0:1787828795.239649:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787828795.239650:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828795.239652:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787828795.239654:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828795/real 1787828795] req@ffff8d30d4d26940 x1874670646765824/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828820 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787828795.239664:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787828795.239665:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787828795.239666:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787828795.239668:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787828795.239668:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828795.239669:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787828795.239671:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787828795.239673:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d26940 x1874670646765824/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828820 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787828795.239678:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828795.239679:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787828795.239680:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828795.239681:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787828795.239682:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828795.239683:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787828795.239684:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787828795.239685:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828795.239687:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787828795.239689:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787828795.239691:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d26940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646765824:192.168.203.143@tcp:8:kworker.0 00000100:00000001:1.0:1787828795.239694:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828795.239695:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787828795.239696:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787828795.239697:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787828795.239698:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787828795.239699:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787828795.239700:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787828795.239702:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787828795.239703:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787828795.239705:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787828795.239706:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828795.239708:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787828795.239709:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787828795.239710:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787828795.239711:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787828795.239712:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787828795.239713:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787828795.239714:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787828795.239715:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787828795.239716:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787828795.239716:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828795.239717:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787828795.239718:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787828795.239719:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787828795.239720:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787828795.239720:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787828795.239721:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787828795.239722:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787828795.239723:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787828795.239724:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787828795.239725:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828795.239726:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828795.239728:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787828795.239729:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828795.239730:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787828795.239731:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828796.134819:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828796.134828:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828796.134944:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828796.134950:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828796.134966:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828797.158744:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828797.158751:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828797.158830:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828797.158833:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828797.158837:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828798.182740:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828798.182748:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828798.182808:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828798.182812:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828798.182815:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828799.206704:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828799.206709:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828799.206785:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828799.206790:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828799.206793:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828800.230669:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828800.230674:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828800.230730:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828800.230733:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828800.230736:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787828800.358793:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828800.358809:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1787828800.358812:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787828800.358817:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787828800.358823:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787828800.358833:0:40798:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 3600 00000100:00080000:0.0:1787828800.358847:0:40798:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787828800.358851:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787828800.358853:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787828800.358858:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787828800.358862:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787828800.358866:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:0.0:1787828800.358868:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:0.0:1787828800.358871:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787828800.358872:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:0.0:1787828800.358878:0:40798:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787828800.358881:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828800.358885:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787828800.358889:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787828800.358893:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828800.358896:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1787828800.358899:0:40798:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828800.358915:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787828800.358917:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828800.358922:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787828800.358925:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787828800.358930:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828800.358948:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828800.358959:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d26940 x1874670646765952/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787828800.358971:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787828800.358976:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828800.359001:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787828800.359005:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828800.359009:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1787828800.359011:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787828800.359012:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787828800.359015:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787828800.359018:0:40798:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3600 00000100:00080000:0.0:1787828800.359022:0:40798:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787828800.359024:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787828800.359026:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787828800.359029:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787828800.359031:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787828800.359033:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000020:00000001:0.0:1787828800.359035:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:0.0:1787828800.359038:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787828800.359040:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:0.0:1787828800.359045:0:40798:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787828800.359048:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828800.359050:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1787828800.359055:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:0.0:1787828800.359058:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787828800.359062:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828800.359062:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1787828800.359066:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787828800.359068:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787828800.359070:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828800.359074:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:0.0:1787828800.359078:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 02000000:00000001:1.0:1787828800.359078:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787828800.359082:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000008:00100000:0.0:1787828800.359086:0:40798:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828800.359088:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d26940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646765952:192.168.203.143@tcp:250:kworker.0 00000008:00000001:0.0:1787828800.359095:0:40798:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828800.359095:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:0.0:1787828800.359100:0:40798:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828800.359103:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787828800.359107:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1787828800.359108:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828800.359111:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828800.359112:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:0.0:1787828800.359116:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787828800.359120:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1787828800.359122:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828800.359123:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828800.359133:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828800.359141:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d25680 x1874670646766080/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787828800.359148:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787828800.359153:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601346880 : -126234108204736 : ffff8d30d4d26940) 00000100:00000001:1.0:1787828800.359160:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:0.0:1787828800.359162:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787828800.359166:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828800.359170:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787828800.359174:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828800.359179:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1787828800.359181:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787828800.359183:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787828800.359186:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787828800.359189:0:40798:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3600 00000100:00080000:0.0:1787828800.359197:0:40798:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787828800.359200:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787828800.359201:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787828800.359204:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787828800.359205:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787828800.359208:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000020:00000001:0.0:1787828800.359214:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000800:00000100:1.0:1787828800.359214:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:0.0:1787828800.359220:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787828800.359222:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:0.0:1787828800.359227:0:40798:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787828800.359231:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828800.359233:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1787828800.359235:0:2390:0:(events.c:34:request_out_callback()) Process entered 02000000:00000001:0.0:1787828800.359237:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787828800.359242:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828800.359242:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828800.359246:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1787828800.359249:0:40798:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00000001:1.0:1787828800.359249:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828800.359253:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000008:00000001:0.0:1787828800.359255:0:40798:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828800.359258:0:40798:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828800.359262:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787828800.359263:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828800.359263:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828800.359268:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787828800.359269:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:1.0:1787828800.359270:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4298274560 02000000:00000001:0.0:1787828800.359271:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828800.359282:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1787828800.359286:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828800.359290:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4298274560 00000100:00100000:0.0:1787828800.359293:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d252c0 x1874670646766208/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787828800.359307:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 00000100:00000001:1.0:1787828800.359313:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787828800.359317:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787828800.359317:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828800.359321:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828800.359322:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00000001:0.0:1787828800.359325:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787828800.359328:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1787828800.359330:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828800/real 1787828800] req@ffff8d30d4d26940 x1874670646765952/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828825 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828800.359332:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1787828800.359335:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787828800.359337:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787828800.359342:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787828800.359347:0:40798:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3600 00000400:00000001:1.0:1787828800.359350:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00080000:0.0:1787828800.359351:0:40798:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000400:00000001:1.0:1787828800.359354:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787828800.359355:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787828800.359358:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:1.0:1787828800.359360:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000020:00000001:0.0:1787828800.359361:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787828800.359364:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:1.0:1787828800.359365:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000020:00000001:0.0:1787828800.359366:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:1.0:1787828800.359368:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1787828800.359371:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:1.0:1787828800.359372:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828800.359375:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787828800.359376:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828800.359379:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00100000:1.0:1787828800.359383:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d26940 x1874670646765952/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828825 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787828800.359386:0:40798:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787828800.359394:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828800.359399:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:0.0:1787828800.359400:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1787828800.359404:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 02000000:00000001:0.0:1787828800.359405:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787828800.359407:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828800.359413:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 02000000:00000001:0.0:1787828800.359414:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828800.359414:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828800.359421:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:1.0:1787828800.359421:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787828800.359425:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000008:00100000:0.0:1787828800.359426:0:40798:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00080000:1.0:1787828800.359428:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000008:00000001:0.0:1787828800.359435:0:40798:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1787828800.359435:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787828800.359440:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000001:0.0:1787828800.359441:0:40798:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787828800.359444:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d26940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646765952:192.168.203.143@tcp:250:kworker.0 02000000:00000001:0.0:1787828800.359450:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:1.0:1787828800.359450:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828800.359455:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 02000000:00000001:0.0:1787828800.359456:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828800.359457:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 00000100:00000001:0.0:1787828800.359461:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1787828800.359462:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787828800.359466:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1787828800.359467:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787828800.359471:0:2390:0:(genops.c:1217:class_import_put()) Process entered 02000000:00000001:0.0:1787828800.359472:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787828800.359473:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 00000100:00000001:0.0:1787828800.359478:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828800.359479:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787828800.359482:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00100000:0.0:1787828800.359501:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d27c00 x1874670646766336/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787828800.359501:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787828800.359503:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828800.359519:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828800.359527:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787828800.359529:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828800.359529:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787828800.359532:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828800.359533:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828800.359537:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787828800.359539:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1787828800.359541:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787828800.359542:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787828800.359543:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828800.359546:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787828800.359547:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787828800.359549:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787828800.359553:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d25680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646766080:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787828800.359556:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787828800.359558:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787828800.359560:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828800.359561:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787828800.359563:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828800.359569:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787828800.359570:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601342080 : -126234108209536 : ffff8d30d4d25680) 00000100:00000001:1.0:1787828800.359587:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1787828800.359598:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787828800.359638:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787828800.359640:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787828800.359641:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828800.359643:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787828800.359645:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828800.359650:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298273303 00000100:00000001:1.0:1787828800.359652:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298274560 00000100:00000001:1.0:1787828800.359654:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298273303 00000100:00000001:1.0:1787828800.359656:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298274560 00000100:00000001:1.0:1787828800.359658:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787828800.359659:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828800.359660:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787828800.359663:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828800/real 1787828800] req@ffff8d30d4d25680 x1874670646766080/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828825 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787828800.359679:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787828800.359680:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787828800.359683:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787828800.359684:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787828800.359686:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828800.359688:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787828800.359689:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787828800.359693:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d25680 x1874670646766080/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828825 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787828800.359701:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828800.359703:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787828800.359706:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828800.359708:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787828800.359709:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828800.359711:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787828800.359713:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787828800.359716:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828800.359721:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787828800.359723:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787828800.359727:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d25680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646766080:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787828800.359731:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787828800.359733:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787828800.359734:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828800.359735:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787828800.359737:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787828800.359738:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787828800.359740:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d252c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646766208:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787828800.359743:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787828800.359744:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787828800.359745:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828800.359747:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787828800.359749:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828800.359754:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787828800.359755:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601341120 : -126234108210496 : ffff8d30d4d252c0) 00000100:00000001:1.0:1787828800.359758:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1787828800.359768:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787828800.359771:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787828800.359773:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787828800.359774:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828800.359775:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787828800.359777:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828800.359780:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298273303 00000100:00000001:1.0:1787828800.359782:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298274560 00000100:00000001:1.0:1787828800.359783:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298273303 00000100:00000001:1.0:1787828800.359785:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298274560 00000100:00000001:1.0:1787828800.359787:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787828800.359788:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828800.359789:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787828800.359792:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828800/real 1787828800] req@ffff8d30d4d252c0 x1874670646766208/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828825 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787828800.359798:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787828800.359799:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787828800.359800:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787828800.359801:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787828800.359802:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828800.359804:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787828800.359805:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787828800.359807:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d252c0 x1874670646766208/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828825 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787828800.359812:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828800.359813:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787828800.359815:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828800.359816:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787828800.359817:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828800.359818:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787828800.359819:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787828800.359821:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828800.359823:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787828800.359824:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787828800.359826:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d252c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646766208:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787828800.359828:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787828800.359829:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787828800.359830:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828800.359831:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787828800.359832:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787828800.359833:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787828800.359835:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d27c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646766336:192.168.203.143@tcp:8:kworker.0 00000100:00000001:1.0:1787828800.359836:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787828800.359838:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787828800.359839:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828800.359840:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787828800.359842:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828800.359846:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787828800.359847:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601351680 : -126234108199936 : ffff8d30d4d27c00) 00000100:00000001:1.0:1787828800.359850:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1787828800.359866:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787828800.359868:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787828800.359870:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787828800.359871:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828800.359872:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787828800.359874:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828800.359875:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298273316 00000100:00000001:1.0:1787828800.359877:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298274560 00000100:00000001:1.0:1787828800.359888:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298273316 00000100:00000001:1.0:1787828800.359890:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298274560 00000100:00000001:1.0:1787828800.359891:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787828800.359893:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828800.359894:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787828800.359897:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828800/real 1787828800] req@ffff8d30d4d27c00 x1874670646766336/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828825 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787828800.359902:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787828800.359903:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787828800.359904:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787828800.359905:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787828800.359906:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828800.359907:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787828800.359908:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787828800.359911:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d27c00 x1874670646766336/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828825 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787828800.359915:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828800.359916:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787828800.359919:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828800.359920:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787828800.359924:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828800.359925:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787828800.359926:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787828800.359928:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828800.359930:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787828800.359932:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787828800.359934:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d27c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646766336:192.168.203.143@tcp:8:kworker.0 00000100:00000001:1.0:1787828800.359936:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828800.359937:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787828800.359938:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787828800.359939:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787828800.359940:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787828800.359941:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787828800.359942:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787828800.359944:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787828800.359945:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787828800.359949:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787828800.359950:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828800.359951:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787828800.359952:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787828800.359952:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787828800.359955:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787828800.359956:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787828800.359956:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787828800.359957:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787828800.359958:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787828800.359959:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787828800.359960:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828800.359972:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787828800.359973:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787828800.359974:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787828800.359975:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787828800.359976:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787828800.359976:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787828800.359977:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787828800.359979:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787828800.359981:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787828800.359982:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828800.359983:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828800.359986:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787828800.359987:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828800.359988:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787828800.359989:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828801.254674:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828801.254679:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828801.254725:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828801.254728:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828801.254731:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828802.278699:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828802.278735:0:815:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1787828802.278739:0:815:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1787828802.278744:0:815:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828802.278750:0:815:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1787828802.278753:0:815:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828802.278757:0:815:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828802.278760:0:815:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828802.278848:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828802.278854:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828802.278858:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828803.302707:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828803.302714:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828803.302783:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828803.302787:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828803.302790:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828804.326668:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828804.326676:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828804.326754:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828804.326758:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828804.326761:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828805.350673:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828805.350679:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828805.350739:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828805.350743:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828805.350746:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787828805.478783:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828805.478811:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1787828805.478816:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1787828805.478821:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787828805.478827:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1787828805.478836:0:468:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 3605 00000100:00080000:1.0:1787828805.478851:0:468:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1787828805.478855:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787828805.478857:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:1.0:1787828805.478863:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:1.0:1787828805.478866:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:1.0:1787828805.478871:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:1.0:1787828805.478873:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:1.0:1787828805.478875:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787828805.478877:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:1.0:1787828805.478882:0:468:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:1.0:1787828805.478885:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828805.478889:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1787828805.478893:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787828805.478897:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828805.478899:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:1.0:1787828805.478902:0:468:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828805.478920:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1787828805.478938:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828805.478942:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1787828805.478946:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1787828805.478952:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828805.478969:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787828805.478984:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d13ef480 x1874670646766464/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787828805.479002:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787828805.479010:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828805.479042:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1787828805.479047:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828805.479052:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1787828805.479055:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1787828805.479058:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787828805.479062:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1787828805.479066:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3605 00000100:00080000:1.0:1787828805.479073:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1787828805.479076:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787828805.479079:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:1.0:1787828805.479082:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:2.0:1787828805.479087:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:1.0:1787828805.479103:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:1.0:1787828805.479109:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:2.0:1787828805.479111:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1787828805.479116:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000020:00000001:1.0:1787828805.479118:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:2.0:1787828805.479122:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787828805.479124:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828805.479126:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828805.479128:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:1.0:1787828805.479140:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 02000000:00000001:2.0:1787828805.479140:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:1.0:1787828805.479147:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828805.479147:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828805.479168:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828805.479174:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1787828805.479183:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00100000:2.0:1787828805.479183:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d13ef480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646766464:192.168.203.143@tcp:250:kworker.0 02000000:00000001:1.0:1787828805.479190:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828805.479194:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:1.0:1787828805.479195:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:1.0:1787828805.479206:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 02000000:00000001:2.0:1787828805.479207:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000008:00000001:1.0:1787828805.479217:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828805.479217:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828805.479251:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:1.0:1787828805.479254:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828805.479260:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828805.479266:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1787828805.479271:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828805.479275:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:2.0:1787828805.479277:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 02000000:00000001:1.0:1787828805.479279:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:2.0:1787828805.479281:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839541351552 : -126234168200064 : ffff8d30d13ef480) 00000100:00000001:2.0:1787828805.479287:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 02000000:00000001:1.0:1787828805.479289:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828805.479301:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787828805.479316:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d13ee580 x1874670646766592/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787828805.479327:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787828805.479343:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787828805.479349:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787828805.479365:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828805.479366:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828805.479373:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828805.479377:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828805.479377:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00080000:1.0:1787828805.479386:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:2.0:1787828805.479386:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1787828805.479398:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1787828805.479398:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4298279680 00000100:00000001:1.0:1787828805.479409:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1787828805.479413:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787828805.479416:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 00000100:00000001:1.0:1787828805.479418:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1787828805.479426:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3605 00000100:00080000:1.0:1787828805.479432:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787828805.479433:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4298279680 00000100:00000001:1.0:1787828805.479439:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828805.479442:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 00000100:00000001:1.0:1787828805.479443:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:2.0:1787828805.479467:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787828805.479472:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828805.479486:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000020:00000001:1.0:1787828805.479494:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:1.0:1787828805.479496:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00100000:2.0:1787828805.479497:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828805/real 1787828805] req@ffff8d30d13ef480 x1874670646766464/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828830 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:1.0:1787828805.479499:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000020:00000001:1.0:1787828805.479506:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:1.0:1787828805.479512:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787828805.479516:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:1.0:1787828805.479528:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000400:00000001:2.0:1787828805.479551:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787828805.479555:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787828805.479556:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828805.479561:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:2.0:1787828805.479562:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1787828805.479569:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 02000000:00000001:1.0:1787828805.479570:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787828805.479572:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:1.0:1787828805.479576:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828805.479578:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787828805.479580:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:2.0:1787828805.479582:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000008:00100000:1.0:1787828805.479584:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1787828805.479594:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787828805.479596:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d13ef480 x1874670646766464/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828830 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787828805.479602:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828805.479609:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1787828805.479612:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828805.479637:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828805.479641:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787828805.479643:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828805.479644:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828805.479645:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828805.479647:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787828805.479649:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787828805.479653:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828805.479656:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:2.0:1787828805.479658:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1787828805.479661:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d13ef480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646766464:192.168.203.143@tcp:250:kworker.0 00000100:00000001:2.0:1787828805.479666:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828805.479669:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828805.479671:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 00000100:00000001:1.0:1787828805.479672:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787828805.479674:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787828805.479677:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787828805.479679:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 02000000:00000001:1.0:1787828805.479681:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1787828805.479686:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:2.0:1787828805.479688:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 00000100:00000001:1.0:1787828805.479693:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828805.479694:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787828805.479700:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00100000:1.0:1787828805.479720:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d13ee1c0 x1874670646766720/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787828805.479722:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787828805.479728:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828805.479731:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828805.479749:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1787828805.479752:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828805.479761:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787828805.479762:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787828805.479770:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1787828805.479771:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828805.479774:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1787828805.479777:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787828805.479780:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828805.479781:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828805.479799:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000100:00080000:1.0:1787828805.479817:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 02000000:00000001:2.0:1787828805.479817:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828805.479828:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1787828805.479831:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1787828805.479836:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00100000:2.0:1787828805.479837:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d13ee580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646766592:192.168.203.143@tcp:38:kworker.0 00000100:00080000:1.0:1787828805.479844:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787828805.479845:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:1.0:1787828805.479856:0:468:0:(import.c:485:import_select_connection()) Process entered 02000000:00000001:2.0:1787828805.479856:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1787828805.479866:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1787828805.479872:0:468:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3605 02000000:00000001:2.0:1787828805.479874:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1787828805.479894:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1787828805.479903:0:468:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787828805.479905:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787828805.479907:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828805.479908:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839541347712 : -126234168203904 : ffff8d30d13ee580) 00000100:00000001:1.0:1787828805.479914:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:2.0:1787828805.479918:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000020:00000001:1.0:1787828805.479919:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:1.0:1787828805.479929:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:1.0:1787828805.479932:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000020:00000001:1.0:1787828805.479935:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:1.0:1787828805.479938:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787828805.479940:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:1.0:1787828805.479944:0:468:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000800:00000100:2.0:1787828805.479951:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787828805.479956:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828805.479960:0:2390:0:(events.c:34:request_out_callback()) Process entered 02000000:00000001:1.0:1787828805.479961:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:2.0:1787828805.479968:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828805.479972:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828805.479973:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787828805.479975:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1787828805.479979:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828805.479982:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828805.479986:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298278423 00000100:00000001:1.0:1787828805.479989:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:2.0:1787828805.479991:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298279680 00000100:00000001:2.0:1787828805.479995:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298278423 00000008:00100000:1.0:1787828805.479999:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00000001:2.0:1787828805.480015:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298279680 00000008:00000001:1.0:1787828805.480028:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828805.480028:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787828805.480048:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828805.480049:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828805.480057:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1787828805.480066:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828805/real 1787828805] req@ffff8d30d13ee580 x1874670646766592/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828830 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:1.0:1787828805.480069:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1787828805.480073:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:2.0:1787828805.480081:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000001:1.0:1787828805.480084:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000400:00000001:2.0:1787828805.480084:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1787828805.480089:0:2390:0:(events.c:78:reply_in_callback()) Process entered 02000000:00000001:1.0:1787828805.480091:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:2.0:1787828805.480092:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787828805.480097:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:1.0:1787828805.480100:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828805.480101:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828805.480103:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787828805.480122:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d13ee580 x1874670646766592/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828830 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787828805.480130:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828805.480132:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828805.480132:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787828805.480143:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828805.480145:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828805.480148:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828805.480152:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00100000:1.0:1787828805.480154:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d13eda40 x1874670646766848/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787828805.480155:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787828805.480167:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828805.480188:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:2.0:1787828805.480196:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1787828805.480209:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d13ee580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646766592:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787828805.480210:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787828805.480220:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828805.480228:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828805.480230:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828805.480234:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828805.480238:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787828805.480239:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1787828805.480253:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787828805.480254:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:2.0:1787828805.480255:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:2.0:1787828805.480259:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787828805.480276:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1787828805.480280:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787828805.480281:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828805.480283:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828805.480287:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1787828805.480288:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828805.480291:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787828805.480293:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1787828805.480294:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1787828805.480296:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787828805.480299:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828805.480303:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787828805.480305:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828805.480307:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787828805.480311:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d13ee1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646766720:192.168.203.143@tcp:38:kworker.0 00000100:00000001:2.0:1787828805.480315:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787828805.480317:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1787828805.480318:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828805.480319:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1787828805.480322:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828805.480326:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787828805.480327:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839541346752 : -126234168204864 : ffff8d30d13ee1c0) 00000100:00000001:2.0:1787828805.480330:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1787828805.480340:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:2.0:1787828805.480343:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1787828805.480345:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828805.480347:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828805.480348:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1787828805.480350:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828805.480353:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298278423 00000100:00000001:2.0:1787828805.480355:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298279680 00000100:00000001:2.0:1787828805.480357:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298278423 00000100:00000001:2.0:1787828805.480359:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298279680 00000100:00000001:2.0:1787828805.480360:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787828805.480362:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828805.480364:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1787828805.480367:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828805/real 1787828805] req@ffff8d30d13ee1c0 x1874670646766720/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828830 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1787828805.480385:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787828805.480386:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1787828805.480388:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1787828805.480389:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787828805.480390:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828805.480391:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828805.480393:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787828805.480395:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d13ee1c0 x1874670646766720/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828830 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787828805.480401:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828805.480402:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787828805.480404:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828805.480405:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828805.480406:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828805.480407:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787828805.480409:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787828805.480426:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828805.480428:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:2.0:1787828805.480429:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1787828805.480431:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d13ee1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646766720:192.168.203.143@tcp:38:kworker.0 00000100:00000001:2.0:1787828805.480434:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1787828805.480435:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787828805.480435:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828805.480437:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787828805.480438:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828805.480465:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787828805.480467:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d13eda40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646766848:192.168.203.143@tcp:8:kworker.0 00000100:00000001:2.0:1787828805.480470:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787828805.480472:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1787828805.480473:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828805.480474:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1787828805.480476:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828805.480481:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787828805.480482:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839541344832 : -126234168206784 : ffff8d30d13eda40) 00000100:00000001:2.0:1787828805.480484:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1787828805.480492:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:2.0:1787828805.480495:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1787828805.480496:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828805.480497:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828805.480498:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1787828805.480500:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828805.480502:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298278436 00000100:00000001:2.0:1787828805.480504:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298279680 00000100:00000001:2.0:1787828805.480516:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298278436 00000100:00000001:2.0:1787828805.480518:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298279680 00000100:00000001:2.0:1787828805.480519:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787828805.480521:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828805.480522:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1787828805.480524:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828805/real 1787828805] req@ffff8d30d13eda40 x1874670646766848/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828830 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1787828805.480529:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787828805.480530:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1787828805.480531:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1787828805.480532:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787828805.480533:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828805.480535:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828805.480536:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787828805.480538:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d13eda40 x1874670646766848/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828830 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787828805.480547:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828805.480548:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787828805.480550:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828805.480551:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828805.480551:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828805.480552:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787828805.480554:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787828805.480555:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828805.480557:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:2.0:1787828805.480558:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1787828805.480560:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d13eda40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646766848:192.168.203.143@tcp:8:kworker.0 00000100:00000001:2.0:1787828805.480562:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828805.480563:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828805.480564:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787828805.480565:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1787828805.480566:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787828805.480567:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:2.0:1787828805.480568:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:2.0:1787828805.480569:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787828805.480570:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1787828805.480571:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787828805.480572:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828805.480573:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828805.480574:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787828805.480575:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1787828805.480576:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787828805.480577:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:2.0:1787828805.480578:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:2.0:1787828805.480579:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787828805.480579:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1787828805.480581:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787828805.480581:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828805.480582:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828805.480584:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1787828805.480586:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828805.480587:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787828805.480588:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828806.374662:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828806.374670:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828806.374736:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828806.374742:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828806.374747:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828807.398725:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828807.398732:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828807.398830:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828807.398834:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828807.398838:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828808.422712:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828808.422719:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828808.422801:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828808.422806:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828808.422810:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828809.446695:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828809.446701:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828809.446764:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828809.446769:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828809.446773:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828810.470734:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828810.470741:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828810.470810:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828810.470816:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828810.470821:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787828810.598783:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828810.598795:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1787828810.598798:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787828810.598802:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787828810.598808:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787828810.598817:0:468:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 3611 00000100:00080000:2.0:1787828810.598830:0:468:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787828810.598834:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828810.598837:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787828810.598841:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787828810.598844:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787828810.598848:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:2.0:1787828810.598851:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:2.0:1787828810.598853:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828810.598855:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:2.0:1787828810.598860:0:468:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787828810.598863:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828810.598867:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787828810.598871:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828810.598874:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828810.598878:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:2.0:1787828810.598881:0:468:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828810.598896:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787828810.598897:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828810.598902:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787828810.598905:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787828810.598910:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828810.598923:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787828810.598934:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2c780 x1874670646766976/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787828810.598947:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787828810.598952:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828810.598981:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787828810.598985:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828810.598989:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1787828810.598991:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787828810.599002:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787828810.599005:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787828810.599007:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3611 00000100:00080000:2.0:1787828810.599011:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787828810.599013:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828810.599015:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787828810.599017:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787828810.599020:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787828810.599023:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000020:00000001:2.0:1787828810.599026:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:2.0:1787828810.599029:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828810.599031:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:2.0:1787828810.599036:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787828810.599040:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828810.599042:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787828810.599050:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828810.599053:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828810.599055:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1787828810.599060:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1787828810.599064:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828810.599067:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828810.599082:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787828810.599088:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828810.599091:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787828810.599092:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787828810.599095:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787828810.599100:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828810.599111:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1787828810.599111:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828810.599122:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00100000:2.0:1787828810.599124:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2fc00 x1874670646767104/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787828810.599130:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787828810.599138:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828810.599155:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 02000000:00000001:1.0:1787828810.599157:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:2.0:1787828810.599162:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828810.599164:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787828810.599174:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828810.599176:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787828810.599196:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828810.599205:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:1.0:1787828810.599206:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646766976:192.168.203.143@tcp:250:kworker.0 00000100:00000001:2.0:1787828810.599208:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787828810.599217:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787828810.599225:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:2.0:1787828810.599227:0:468:0:(import.c:485:import_select_connection()) Process entered 02000000:00000001:1.0:1787828810.599238:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000100:00080000:2.0:1787828810.599240:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3611 02000000:00000001:1.0:1787828810.599251:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787828810.599254:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 02000000:00000001:1.0:1787828810.599258:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:2.0:1787828810.599260:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828810.599266:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787828810.599270:0:468:0:(genops.c:792:class_conn2export()) Process entered 02000000:00000001:1.0:1787828810.599275:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1787828810.599276:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787828810.599285:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000020:00000001:2.0:1787828810.599291:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:2.0:1787828810.599295:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828810.599298:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:2.0:1787828810.599305:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787828810.599307:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828810.599310:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 02000000:00000001:2.0:1787828810.599310:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1787828810.599322:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417870208 : -126234291681408 : ffff8d30c9e2c780) 02000000:00000001:2.0:1787828810.599324:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828810.599336:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828810.599337:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:2.0:1787828810.599348:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1787828810.599351:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1787828810.599355:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828810.599358:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828810.599363:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787828810.599365:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828810.599368:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787828810.599370:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787828810.599374:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828810.599378:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787828810.599384:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2c3c0 x1874670646767232/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787828810.599407:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787828810.599444:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828810.599449:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787828810.599459:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:1.0:1787828810.599462:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00080000:2.0:1787828810.599472:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1787828810.599478:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787828810.599481:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787828810.599486:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:1.0:1787828810.599495:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00080000:2.0:1787828810.599497:0:468:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3611 00000100:00080000:2.0:1787828810.599508:0:468:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1787828810.599510:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828810.599513:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787828810.599516:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828810.599518:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:1.0:1787828810.599523:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000020:00000001:2.0:1787828810.599535:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787828810.599539:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:1.0:1787828810.599546:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1787828810.599548:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000020:00000001:2.0:1787828810.599564:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:1.0:1787828810.599567:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4298284800 00000100:00000001:2.0:1787828810.599569:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828810.599577:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:2.0:1787828810.599583:0:468:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787828810.599587:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828810.599589:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 02000000:00000001:2.0:1787828810.599590:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1787828810.599609:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4298284800 02000000:00000001:2.0:1787828810.599657:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787828810.599665:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 02000000:00000001:2.0:1787828810.599669:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828810.599673:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787828810.599674:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1787828810.599681:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00000001:1.0:1787828810.599684:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:2.0:1787828810.599690:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828810.599692:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00000001:2.0:1787828810.599702:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828810.599713:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00100000:1.0:1787828810.599715:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828810/real 1787828810] req@ffff8d30c9e2c780 x1874670646766976/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828835 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1787828810.599725:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828810.599742:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787828810.599748:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787828810.599760:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:1.0:1787828810.599761:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787828810.599770:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1787828810.599777:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828810.599779:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787828810.599789:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787828810.599796:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787828810.599798:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2de00 x1874670646767360/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787828810.599807:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787828810.599814:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828810.599832:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00100000:1.0:1787828810.599837:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2c780 x1874670646766976/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828835 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787828810.599838:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828810.599854:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828810.599873:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828810.599880:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787828810.599885:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828810.599887:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787828810.599890:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828810.599893:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787828810.599897:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787828810.599902:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828810.599912:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787828810.599915:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787828810.599921:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646766976:192.168.203.143@tcp:250:kworker.0 00000100:00000001:1.0:1787828810.599927:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828810.599930:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787828810.599932:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787828810.599935:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787828810.599941:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787828810.599945:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787828810.599948:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787828810.599954:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787828810.599956:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787828810.599961:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787828810.599963:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828810.599967:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828810.599979:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787828810.599983:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828810.599988:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787828810.600016:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1787828810.600018:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787828810.600021:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787828810.600023:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828810.600028:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787828810.600032:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787828810.600036:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787828810.600042:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646767104:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787828810.600050:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787828810.600054:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787828810.600057:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828810.600060:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787828810.600082:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828810.600094:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787828810.600097:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417883648 : -126234291667968 : ffff8d30c9e2fc00) 00000100:00000001:1.0:1787828810.600105:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1787828810.600126:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787828810.600133:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787828810.600137:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787828810.600139:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828810.600141:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787828810.600145:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828810.600152:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298283543 00000100:00000001:1.0:1787828810.600157:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298284800 00000100:00000001:1.0:1787828810.600162:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298283543 00000100:00000001:1.0:1787828810.600166:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298284800 00000100:00000001:1.0:1787828810.600171:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787828810.600175:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828810.600178:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787828810.600187:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828810/real 1787828810] req@ffff8d30c9e2fc00 x1874670646767104/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828835 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787828810.600205:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787828810.600207:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787828810.600213:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787828810.600215:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787828810.600218:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828810.600222:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787828810.600225:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787828810.600232:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2fc00 x1874670646767104/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828835 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787828810.600246:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828810.600251:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787828810.600255:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828810.600258:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787828810.600261:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828810.600264:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787828810.600268:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787828810.600273:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828810.600279:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787828810.600284:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787828810.600291:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646767104:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787828810.600298:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787828810.600302:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787828810.600305:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828810.600309:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787828810.600312:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787828810.600316:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787828810.600322:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646767232:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787828810.600328:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787828810.600333:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787828810.600336:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828810.600339:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787828810.600345:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828810.600355:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787828810.600357:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417869248 : -126234291682368 : ffff8d30c9e2c3c0) 00000100:00000001:1.0:1787828810.600364:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1787828810.600386:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787828810.600394:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787828810.600398:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787828810.600401:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828810.600404:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787828810.600409:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828810.600438:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298283543 00000100:00000001:1.0:1787828810.600444:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298284800 00000100:00000001:1.0:1787828810.600449:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298283543 00000100:00000001:1.0:1787828810.600452:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298284800 00000100:00000001:1.0:1787828810.600457:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787828810.600461:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828810.600466:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787828810.600475:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828810/real 1787828810] req@ffff8d30c9e2c3c0 x1874670646767232/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828835 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787828810.600505:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787828810.600508:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787828810.600513:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787828810.600516:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787828810.600518:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828810.600522:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787828810.600525:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787828810.600533:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2c3c0 x1874670646767232/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828835 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787828810.600548:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828810.600552:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787828810.600556:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828810.600560:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787828810.600563:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828810.600567:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787828810.600569:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787828810.600574:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828810.600580:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787828810.600586:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787828810.600592:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646767232:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787828810.600639:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787828810.600643:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787828810.600646:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828810.600651:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787828810.600655:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787828810.600659:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787828810.600665:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646767360:192.168.203.143@tcp:8:kworker.0 00000100:00000001:1.0:1787828810.600672:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787828810.600676:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787828810.600681:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828810.600685:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787828810.600691:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828810.600701:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787828810.600704:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417875968 : -126234291675648 : ffff8d30c9e2de00) 00000100:00000001:1.0:1787828810.600711:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1787828810.600735:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787828810.600744:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787828810.600749:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787828810.600752:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828810.600755:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787828810.600761:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828810.600767:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298283557 00000100:00000001:1.0:1787828810.600773:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298284800 00000100:00000001:1.0:1787828810.600777:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298283557 00000100:00000001:1.0:1787828810.600782:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298284800 00000100:00000001:1.0:1787828810.600786:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787828810.600790:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828810.600794:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787828810.600803:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828810/real 1787828810] req@ffff8d30c9e2de00 x1874670646767360/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828835 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787828810.600819:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787828810.600821:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787828810.600827:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787828810.600830:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787828810.600833:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828810.600836:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787828810.600839:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787828810.600847:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2de00 x1874670646767360/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828835 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787828810.600860:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828810.600864:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787828810.600870:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828810.600873:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787828810.600875:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828810.600879:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787828810.600882:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787828810.600887:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828810.600895:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787828810.600899:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787828810.600905:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646767360:192.168.203.143@tcp:8:kworker.0 00000100:00000001:1.0:1787828810.600913:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828810.600916:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787828810.600919:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787828810.600922:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787828810.600926:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787828810.600929:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787828810.600932:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787828810.600936:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787828810.600939:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787828810.600944:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787828810.600947:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828810.600951:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787828810.600953:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787828810.600956:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787828810.600959:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787828810.600962:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787828810.600964:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787828810.600968:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787828810.600970:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787828810.600973:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787828810.600993:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828810.600997:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787828810.601006:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787828810.601009:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787828810.601012:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787828810.601014:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787828810.601017:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787828810.601020:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787828810.601023:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787828810.601033:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787828810.601035:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828810.601039:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828810.601046:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787828810.601049:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828810.601053:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787828810.601056:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828811.494739:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828811.494758:0:815:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1787828811.494762:0:815:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1787828811.494766:0:815:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828811.494773:0:815:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1787828811.494776:0:815:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828811.494779:0:815:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828811.494782:0:815:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828811.494862:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828811.494866:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828811.494870:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828812.518779:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828812.518788:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828812.518898:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828812.518904:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828812.518908:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828813.542683:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828813.542691:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828813.542753:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828813.542757:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828813.542762:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828814.566689:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828814.566694:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828814.566748:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828814.566752:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828814.566755:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828815.590760:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828815.590768:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828815.590832:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828815.590837:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828815.590842:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787828815.718749:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828815.718758:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1787828815.718761:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1787828815.718764:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787828815.718769:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1787828815.718775:0:468:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 3616 00000100:00080000:1.0:1787828815.718785:0:468:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1787828815.718787:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787828815.718789:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:1.0:1787828815.718793:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:1.0:1787828815.718795:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:1.0:1787828815.718798:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:1.0:1787828815.718800:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:1.0:1787828815.718802:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787828815.718803:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:1.0:1787828815.718807:0:468:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:1.0:1787828815.718810:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828815.718812:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1787828815.718815:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787828815.718818:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828815.718825:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:1.0:1787828815.718827:0:468:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828815.718840:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1787828815.718842:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828815.718846:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1787828815.718849:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1787828815.718855:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828815.718869:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787828815.718880:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d13ecf00 x1874670646767488/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787828815.718894:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787828815.718900:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828815.718918:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1787828815.718920:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828815.718924:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1787828815.718925:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1787828815.718926:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787828815.718930:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1787828815.718933:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3616 00000100:00080000:1.0:1787828815.718937:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1787828815.718939:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787828815.718941:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:1.0:1787828815.718944:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:1.0:1787828815.718945:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:1.0:1787828815.718947:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:2.0:1787828815.718948:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:1.0:1787828815.718950:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:1.0:1787828815.718953:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787828815.718955:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:2.0:1787828815.718955:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1787828815.718960:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00080000:1.0:1787828815.718961:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787828815.718964:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787828815.718965:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828815.718968:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828815.718969:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1787828815.718976:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828815.718976:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787828815.718980:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787828815.718981:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828815.718988:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 02000000:00000001:2.0:1787828815.718988:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000008:00100000:1.0:1787828815.718993:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828815.718997:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d13ecf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646767488:192.168.203.143@tcp:250:kworker.0 00000008:00000001:1.0:1787828815.718998:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828815.719004:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828815.719007:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787828815.719011:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787828815.719013:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787828815.719014:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828815.719035:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787828815.719035:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828815.719038:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787828815.719040:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787828815.719041:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828815.719047:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828815.719049:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787828815.719056:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d13eda40 x1874670646767616/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787828815.719062:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787828815.719063:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787828815.719065:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828815.719066:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839541341952 : -126234168209664 : ffff8d30d13ecf00) 00000100:00000001:1.0:1787828815.719069:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828815.719071:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00080000:1.0:1787828815.719073:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828815.719077:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:1.0:1787828815.719078:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1787828815.719079:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787828815.719082:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1787828815.719084:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3616 00000100:00080000:1.0:1787828815.719088:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1787828815.719089:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787828815.719090:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:1.0:1787828815.719091:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:1.0:1787828815.719093:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:1.0:1787828815.719095:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000020:00000001:1.0:1787828815.719103:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:1.0:1787828815.719106:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787828815.719109:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000800:00000100:2.0:1787828815.719109:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00080000:1.0:1787828815.719116:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:1.0:1787828815.719119:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828815.719121:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:2.0:1787828815.719123:0:2390:0:(events.c:34:request_out_callback()) Process entered 02000000:00000001:1.0:1787828815.719124:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787828815.719128:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828815.719130:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787828815.719131:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:1.0:1787828815.719135:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00000001:2.0:1787828815.719136:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:1.0:1787828815.719141:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828815.719141:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787828815.719146:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828815.719148:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828815.719150:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1787828815.719153:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828815.719154:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:2.0:1787828815.719154:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4298289920 02000000:00000001:1.0:1787828815.719157:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1787828815.719160:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828815.719164:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828815.719166:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 00000100:00100000:1.0:1787828815.719173:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d13ee1c0 x1874670646767744/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787828815.719180:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787828815.719182:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828815.719183:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4298289920 00000100:00000001:1.0:1787828815.719184:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1787828815.719189:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828815.719192:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1787828815.719193:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 00000100:00000001:1.0:1787828815.719194:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1787828815.719197:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787828815.719200:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:2.0:1787828815.719202:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00080000:1.0:1787828815.719203:0:468:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3616 00000100:00080000:1.0:1787828815.719210:0:468:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787828815.719210:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828815.719214:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787828815.719217:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:2.0:1787828815.719217:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000020:00000001:1.0:1787828815.719221:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:1.0:1787828815.719223:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:1.0:1787828815.719225:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000020:00000001:1.0:1787828815.719228:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:1.0:1787828815.719230:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787828815.719231:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00100000:2.0:1787828815.719231:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828815/real 1787828815] req@ffff8d30d13ecf00 x1874670646767488/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828840 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828815.719242:0:468:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:1.0:1787828815.719249:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828815.719250:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000400:00000001:2.0:1787828815.719251:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 02000000:00000001:1.0:1787828815.719253:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000001:2.0:1787828815.719254:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 02000000:00000001:1.0:1787828815.719256:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828815.719259:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:2.0:1787828815.719260:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000008:00100000:1.0:1787828815.719261:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000008:00000001:1.0:1787828815.719265:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828815.719265:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787828815.719268:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828815.719269:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:1.0:1787828815.719272:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:2.0:1787828815.719273:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 02000000:00000001:1.0:1787828815.719275:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828815.719276:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828815.719278:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1787828815.719280:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1787828815.719283:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787828815.719284:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d13ecf00 x1874670646767488/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828840 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787828815.719287:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787828815.719295:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d13ee580 x1874670646767872/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787828815.719297:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828815.719302:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787828815.719309:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828815.719310:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787828815.719314:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828815.719314:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828815.719317:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828815.719319:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828815.719322:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787828815.719325:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787828815.719328:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828815.719331:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:2.0:1787828815.719333:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1787828815.719337:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d13ecf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646767488:192.168.203.143@tcp:250:kworker.0 00000100:00000001:2.0:1787828815.719341:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828815.719342:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828815.719344:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787828815.719345:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1787828815.719348:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787828815.719349:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:2.0:1787828815.719351:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:2.0:1787828815.719354:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787828815.719355:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1787828815.719358:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787828815.719359:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828815.719361:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828815.719366:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1787828815.719368:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828815.719370:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787828815.719371:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1787828815.719372:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1787828815.719373:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787828815.719375:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828815.719376:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787828815.719378:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828815.719395:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787828815.719398:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d13eda40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646767616:192.168.203.143@tcp:38:kworker.0 00000100:00000001:2.0:1787828815.719400:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787828815.719402:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1787828815.719403:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828815.719405:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1787828815.719407:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828815.719412:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787828815.719430:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839541344832 : -126234168206784 : ffff8d30d13eda40) 00000100:00000001:2.0:1787828815.719435:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1787828815.719453:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:2.0:1787828815.719460:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1787828815.719463:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828815.719465:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828815.719467:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1787828815.719470:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828815.719475:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298288662 00000100:00000001:2.0:1787828815.719479:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298289920 00000100:00000001:2.0:1787828815.719483:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298288662 00000100:00000001:2.0:1787828815.719485:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298289920 00000100:00000001:2.0:1787828815.719488:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787828815.719491:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828815.719493:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1787828815.719499:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828815/real 1787828815] req@ffff8d30d13eda40 x1874670646767616/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828840 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1787828815.719509:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787828815.719511:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1787828815.719513:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1787828815.719515:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787828815.719516:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828815.719517:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828815.719519:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787828815.719522:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d13eda40 x1874670646767616/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828840 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787828815.719528:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828815.719530:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787828815.719532:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828815.719533:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828815.719534:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828815.719536:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787828815.719537:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787828815.719539:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828815.719542:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:2.0:1787828815.719543:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1787828815.719546:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d13eda40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646767616:192.168.203.143@tcp:38:kworker.0 00000100:00000001:2.0:1787828815.719548:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1787828815.719549:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787828815.719551:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828815.719569:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787828815.719570:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828815.719573:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787828815.719576:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d13ee1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646767744:192.168.203.143@tcp:38:kworker.0 00000100:00000001:2.0:1787828815.719579:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787828815.719581:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1787828815.719584:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828815.719586:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1787828815.719589:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828815.719595:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787828815.719596:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839541346752 : -126234168204864 : ffff8d30d13ee1c0) 00000100:00000001:2.0:1787828815.719601:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1787828815.719615:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:2.0:1787828815.719620:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1787828815.719623:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828815.719624:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828815.719626:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1787828815.719629:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828815.719650:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298288663 00000100:00000001:2.0:1787828815.719652:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298289920 00000100:00000001:2.0:1787828815.719654:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298288663 00000100:00000001:2.0:1787828815.719656:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298289920 00000100:00000001:2.0:1787828815.719657:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787828815.719659:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828815.719660:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1787828815.719664:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828815/real 1787828815] req@ffff8d30d13ee1c0 x1874670646767744/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828840 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1787828815.719669:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787828815.719670:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1787828815.719672:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1787828815.719673:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787828815.719674:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828815.719676:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828815.719677:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787828815.719680:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d13ee1c0 x1874670646767744/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828840 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787828815.719684:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828815.719686:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787828815.719688:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828815.719689:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828815.719690:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828815.719691:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787828815.719709:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787828815.719711:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828815.719714:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:2.0:1787828815.719715:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1787828815.719718:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d13ee1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646767744:192.168.203.143@tcp:38:kworker.0 00000100:00000001:2.0:1787828815.719720:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1787828815.719721:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787828815.719722:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828815.719724:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787828815.719726:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828815.719727:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787828815.719730:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d13ee580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646767872:192.168.203.143@tcp:8:kworker.0 00000100:00000001:2.0:1787828815.719732:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787828815.719734:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1787828815.719735:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828815.719737:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1787828815.719739:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828815.719744:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787828815.719745:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839541347712 : -126234168203904 : ffff8d30d13ee580) 00000100:00000001:2.0:1787828815.719748:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1787828815.719758:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:2.0:1787828815.719761:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1787828815.719763:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828815.719765:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828815.719766:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1787828815.719768:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828815.719771:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298288676 00000100:00000001:2.0:1787828815.719772:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298289920 00000100:00000001:2.0:1787828815.719775:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298288676 00000100:00000001:2.0:1787828815.719776:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298289920 00000100:00000001:2.0:1787828815.719778:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787828815.719780:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828815.719781:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1787828815.719784:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828815/real 1787828815] req@ffff8d30d13ee580 x1874670646767872/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828840 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1787828815.719790:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787828815.719791:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1787828815.719792:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1787828815.719794:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787828815.719795:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828815.719796:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828815.719797:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787828815.719800:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d13ee580 x1874670646767872/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828840 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787828815.719805:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828815.719807:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787828815.719809:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828815.719810:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828815.719811:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828815.719812:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787828815.719813:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787828815.719815:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828815.719817:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:2.0:1787828815.719819:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1787828815.719821:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d13ee580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646767872:192.168.203.143@tcp:8:kworker.0 00000100:00000001:2.0:1787828815.719824:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828815.719836:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828815.719838:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787828815.719840:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1787828815.719842:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787828815.719843:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:2.0:1787828815.719845:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:2.0:1787828815.719847:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787828815.719848:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1787828815.719850:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787828815.719851:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828815.719853:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828815.719854:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787828815.719855:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1787828815.719856:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787828815.719857:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:2.0:1787828815.719858:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:2.0:1787828815.719859:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787828815.719860:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1787828815.719861:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787828815.719862:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828815.719863:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828815.719864:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787828815.719865:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1787828815.719866:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787828815.719867:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:2.0:1787828815.719868:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:2.0:1787828815.719869:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787828815.719870:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1787828815.719872:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787828815.719873:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828815.719874:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828815.719877:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1787828815.719878:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828815.719880:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787828815.719881:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828816.614732:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828816.614740:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828816.614843:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828816.614849:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828816.614872:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828817.638726:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828817.638735:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828817.638820:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828817.638829:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828817.638837:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828818.662738:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828818.662746:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828818.662825:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828818.662830:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828818.662834:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828819.686748:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828819.686760:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828819.686838:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828819.686843:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828819.686847:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828820.710787:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828820.710795:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828820.710879:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828820.710885:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828820.710890:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787828820.838760:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828820.838772:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1787828820.838775:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787828820.838781:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787828820.838788:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787828820.838799:0:468:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 3621 00000100:00080000:2.0:1787828820.838835:0:468:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787828820.838840:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828820.838843:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787828820.838849:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787828820.838852:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787828820.838856:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:2.0:1787828820.838859:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:2.0:1787828820.838862:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828820.838864:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:2.0:1787828820.838870:0:468:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787828820.838874:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828820.838878:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787828820.838883:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828820.838887:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828820.838891:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:2.0:1787828820.838894:0:468:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828820.838912:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787828820.838915:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828820.838920:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787828820.838923:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787828820.838945:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828820.838973:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787828820.838985:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2e940 x1874670646768000/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787828820.839001:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787828820.839006:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828820.839052:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787828820.839055:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828820.839058:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1787828820.839060:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787828820.839061:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787828820.839063:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787828820.839065:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3621 00000100:00080000:2.0:1787828820.839069:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787828820.839072:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828820.839074:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787828820.839076:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787828820.839078:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787828820.839080:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000020:00000001:2.0:1787828820.839082:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:1.0:1787828820.839083:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787828820.839084:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828820.839089:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:1.0:1787828820.839091:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00080000:2.0:1787828820.839092:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:1.0:1787828820.839096:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1787828820.839098:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828820.839101:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1787828820.839102:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787828820.839105:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828820.839108:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787828820.839110:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787828820.839114:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828820.839115:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787828820.839117:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 02000000:00000001:1.0:1787828820.839120:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000008:00100000:2.0:1787828820.839124:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787828820.839128:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646768000:192.168.203.143@tcp:250:kworker.0 00000008:00000001:2.0:1787828820.839129:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828820.839134:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828820.839139:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787828820.839141:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787828820.839144:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828820.839145:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:2.0:1787828820.839148:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1787828820.839150:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828820.839151:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1787828820.839156:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1787828820.839156:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828820.839164:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828820.839165:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787828820.839171:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2f0c0 x1874670646768128/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787828820.839179:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787828820.839180:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828820.839183:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787828820.839186:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828820.839188:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1787828820.839189:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787828820.839190:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787828820.839192:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:1.0:1787828820.839195:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00080000:2.0:1787828820.839195:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3621 00000100:00000001:1.0:1787828820.839200:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417878848 : -126234291672768 : ffff8d30c9e2e940) 00000100:00080000:2.0:1787828820.839200:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787828820.839205:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787828820.839208:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:2.0:1787828820.839208:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787828820.839223:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787828820.839224:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787828820.839226:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000020:00000001:2.0:1787828820.839228:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:2.0:1787828820.839229:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828820.839230:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:2.0:1787828820.839232:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787828820.839234:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828820.839236:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787828820.839238:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828820.839240:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828820.839241:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1787828820.839243:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1787828820.839245:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828820.839247:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828820.839250:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787828820.839251:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828820.839252:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787828820.839254:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787828820.839255:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828820.839260:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787828820.839262:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2f480 x1874670646768256/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787828820.839272:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:2.0:1787828820.839273:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787828820.839278:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828820.839279:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787828820.839281:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828820.839284:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1787828820.839285:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787828820.839286:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787828820.839289:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:1.0:1787828820.839290:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00080000:2.0:1787828820.839292:0:468:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3621 00000100:00080000:2.0:1787828820.839297:0:468:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1787828820.839298:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828820.839300:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828820.839303:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787828820.839304:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787828820.839305:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787828820.839307:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:1.0:1787828820.839309:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1787828820.839309:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:1.0:1787828820.839316:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1787828820.839317:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828820.839320:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:2.0:1787828820.839324:0:468:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787828820.839327:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828820.839328:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828820.839334:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1787828820.839337:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4298295040 02000000:00000001:2.0:1787828820.839338:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828820.839357:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828820.839360:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1787828820.839362:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00000001:1.0:1787828820.839363:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 00000008:00000001:2.0:1787828820.839364:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828820.839370:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828820.839382:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787828820.839383:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828820.839385:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787828820.839386:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:1.0:1787828820.839388:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4298295040 02000000:00000001:2.0:1787828820.839389:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828820.839397:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 00000100:00000001:2.0:1787828820.839399:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828820.839402:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787828820.839408:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787828820.839408:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2c000 x1874670646768384/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787828820.839413:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00000001:2.0:1787828820.839420:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00100000:1.0:1787828820.839422:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828820/real 1787828820] req@ffff8d30c9e2e940 x1874670646768000/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828845 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787828820.839424:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828820.839429:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:1.0:1787828820.839441:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787828820.839443:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787828820.839447:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787828820.839450:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787828820.839452:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828820.839455:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787828820.839457:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787828820.839461:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2e940 x1874670646768000/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828845 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787828820.839468:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828820.839471:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787828820.839474:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828820.839475:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787828820.839476:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828820.839478:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787828820.839479:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787828820.839482:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828820.839486:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787828820.839489:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787828820.839492:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646768000:192.168.203.143@tcp:250:kworker.0 00000100:00000001:1.0:1787828820.839497:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828820.839498:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787828820.839500:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787828820.839502:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787828820.839505:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787828820.839506:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787828820.839508:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787828820.839512:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787828820.839514:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787828820.839518:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787828820.839519:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828820.839521:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828820.839529:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787828820.839531:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828820.839533:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787828820.839534:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1787828820.839535:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787828820.839536:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787828820.839538:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828820.839540:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787828820.839541:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787828820.839542:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787828820.839545:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646768128:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787828820.839548:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787828820.839549:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787828820.839551:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828820.839552:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787828820.839555:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828820.839559:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787828820.839561:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417880768 : -126234291670848 : ffff8d30c9e2f0c0) 00000100:00000001:1.0:1787828820.839566:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1787828820.839575:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787828820.839579:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787828820.839581:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787828820.839582:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828820.839583:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787828820.839585:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828820.839589:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298293782 00000100:00000001:1.0:1787828820.839592:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298295040 00000100:00000001:1.0:1787828820.839594:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298293782 00000100:00000001:1.0:1787828820.839596:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298295040 00000100:00000001:1.0:1787828820.839598:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787828820.839619:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828820.839620:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787828820.839624:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828820/real 1787828820] req@ffff8d30c9e2f0c0 x1874670646768128/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828845 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787828820.839630:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787828820.839632:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787828820.839634:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787828820.839635:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787828820.839636:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828820.839637:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787828820.839639:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787828820.839642:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2f0c0 x1874670646768128/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828845 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787828820.839647:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828820.839648:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787828820.839650:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828820.839651:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787828820.839652:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828820.839653:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787828820.839654:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787828820.839657:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828820.839659:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787828820.839662:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787828820.839664:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646768128:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787828820.839667:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787828820.839668:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787828820.839669:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828820.839671:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787828820.839672:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787828820.839673:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787828820.839675:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646768256:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787828820.839678:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787828820.839679:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787828820.839680:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828820.839682:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787828820.839685:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828820.839688:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787828820.839690:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417881728 : -126234291669888 : ffff8d30c9e2f480) 00000100:00000001:1.0:1787828820.839692:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1787828820.839701:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787828820.839704:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787828820.839705:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787828820.839707:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828820.839708:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787828820.839710:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828820.839712:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298293783 00000100:00000001:1.0:1787828820.839715:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298295040 00000100:00000001:1.0:1787828820.839716:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298293783 00000100:00000001:1.0:1787828820.839718:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298295040 00000100:00000001:1.0:1787828820.839720:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787828820.839722:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828820.839723:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787828820.839725:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828820/real 1787828820] req@ffff8d30c9e2f480 x1874670646768256/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828845 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787828820.839737:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787828820.839738:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787828820.839740:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787828820.839741:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787828820.839742:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828820.839743:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787828820.839744:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787828820.839746:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2f480 x1874670646768256/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828845 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787828820.839751:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828820.839753:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787828820.839755:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828820.839756:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787828820.839758:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828820.839759:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787828820.839760:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787828820.839761:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828820.839764:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787828820.839765:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787828820.839767:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646768256:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787828820.839770:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787828820.839770:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787828820.839771:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828820.839773:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787828820.839774:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787828820.839775:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787828820.839777:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646768384:192.168.203.143@tcp:8:kworker.0 00000100:00000001:1.0:1787828820.839779:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787828820.839780:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787828820.839782:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828820.839783:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787828820.839786:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828820.839790:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787828820.839791:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417868288 : -126234291683328 : ffff8d30c9e2c000) 00000100:00000001:1.0:1787828820.839794:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1787828820.839800:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787828820.839803:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787828820.839804:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787828820.839806:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828820.839807:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787828820.839809:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828820.839811:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298293796 00000100:00000001:1.0:1787828820.839812:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298295040 00000100:00000001:1.0:1787828820.839814:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298293796 00000100:00000001:1.0:1787828820.839816:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298295040 00000100:00000001:1.0:1787828820.839818:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787828820.839819:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828820.839821:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787828820.839823:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828820/real 1787828820] req@ffff8d30c9e2c000 x1874670646768384/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828845 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787828820.839828:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787828820.839829:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787828820.839830:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787828820.839831:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787828820.839832:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828820.839833:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787828820.839834:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787828820.839837:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2c000 x1874670646768384/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828845 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787828820.839841:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828820.839842:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787828820.839845:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828820.839846:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787828820.839847:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828820.839848:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787828820.839850:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787828820.839852:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828820.839854:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787828820.839855:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787828820.839857:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646768384:192.168.203.143@tcp:8:kworker.0 00000100:00000001:1.0:1787828820.839859:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828820.839861:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787828820.839862:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787828820.839871:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787828820.839873:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787828820.839873:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787828820.839874:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787828820.839876:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787828820.839877:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787828820.839880:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787828820.839881:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828820.839882:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787828820.839883:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787828820.839884:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787828820.839885:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787828820.839886:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787828820.839887:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787828820.839888:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787828820.839888:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787828820.839890:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787828820.839891:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828820.839892:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787828820.839893:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787828820.839894:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787828820.839896:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787828820.839896:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787828820.839897:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787828820.839898:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787828820.839900:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787828820.839902:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787828820.839903:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828820.839904:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828820.839907:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787828820.839908:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828820.839909:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787828820.839911:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828821.734791:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828821.734801:0:815:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1787828821.734804:0:815:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1787828821.734808:0:815:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828821.734813:0:815:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1787828821.734816:0:815:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828821.734818:0:815:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828821.734820:0:815:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828821.734929:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828821.734940:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828821.734947:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828822.758753:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828822.758760:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828822.758830:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828822.758835:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828822.758839:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828823.782703:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828823.782709:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828823.782770:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828823.782777:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828823.782782:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828824.806736:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828824.806741:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828824.806801:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828824.806805:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828824.806809:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828825.830699:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828825.830705:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828825.830799:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828825.830803:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828825.830806:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787828825.958743:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828825.958751:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1787828825.958754:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787828825.958758:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787828825.958762:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787828825.958770:0:468:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 3626 00000100:00080000:2.0:1787828825.958780:0:468:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787828825.958782:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828825.958785:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787828825.958789:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787828825.958792:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787828825.958794:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:2.0:1787828825.958796:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:2.0:1787828825.958797:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828825.958800:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:2.0:1787828825.958803:0:468:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787828825.958805:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828825.958807:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787828825.958811:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828825.958814:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828825.958816:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:2.0:1787828825.958818:0:468:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828825.958826:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787828825.958827:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828825.958830:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787828825.958832:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787828825.958835:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828825.958845:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787828825.958852:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2ed00 x1874670646768512/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787828825.958860:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787828825.958864:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828825.958881:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787828825.958884:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828825.958889:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1787828825.958890:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787828825.958892:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787828825.958894:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787828825.958896:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3626 00000100:00080000:2.0:1787828825.958899:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787828825.958902:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828825.958904:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787828825.958906:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787828825.958908:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787828825.958910:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000020:00000001:2.0:1787828825.958913:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:2.0:1787828825.958915:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828825.958916:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:2.0:1787828825.958919:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787828825.958921:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828825.958923:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787828825.958927:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787828825.958930:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:2.0:1787828825.958930:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828825.958935:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:1.0:1787828825.958937:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000008:00100000:2.0:1787828825.958939:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00000001:1.0:1787828825.958941:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000008:00000001:2.0:1787828825.958950:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828825.958951:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787828825.958960:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828825.958960:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828825.958976:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1787828825.958977:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787828825.958984:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828825.958985:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787828825.958993:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828825.958993:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787828825.959002:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787828825.959005:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787828825.959009:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646768512:192.168.203.143@tcp:250:kworker.0 00000100:00000001:2.0:1787828825.959011:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828825.959020:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787828825.959028:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787828825.959033:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787828825.959033:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2da40 x1874670646768640/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:1.0:1787828825.959042:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787828825.959066:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828825.959071:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787828825.959076:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828825.959082:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787828825.959085:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828825.959088:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1787828825.959090:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787828825.959093:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787828825.959096:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787828825.959098:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3626 00000100:00000001:1.0:1787828825.959100:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00080000:2.0:1787828825.959102:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1787828825.959106:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417879808 : -126234291671808 : ffff8d30c9e2ed00) 00000100:00000001:2.0:1787828825.959107:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828825.959110:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787828825.959116:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787828825.959118:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:1.0:1787828825.959119:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000020:00000001:2.0:1787828825.959120:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000020:00000001:2.0:1787828825.959124:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:2.0:1787828825.959126:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828825.959128:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:2.0:1787828825.959130:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787828825.959132:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828825.959133:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787828825.959137:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828825.959139:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828825.959140:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1787828825.959142:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1787828825.959143:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828825.959145:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828825.959148:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787828825.959149:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828825.959151:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787828825.959152:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787828825.959153:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828825.959158:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787828825.959169:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2c000 x1874670646768768/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787828825.959176:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787828825.959177:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828825.959179:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787828825.959182:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828825.959184:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1787828825.959186:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787828825.959187:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787828825.959189:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787828825.959191:0:468:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3626 00000100:00080000:2.0:1787828825.959194:0:468:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1787828825.959195:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828825.959196:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828825.959203:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787828825.959206:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:1.0:1787828825.959208:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4298300160 00000020:00000001:2.0:1787828825.959208:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787828825.959212:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000020:00000001:2.0:1787828825.959215:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:2.0:1787828825.959217:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828825.959219:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:1.0:1787828825.959221:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 00000100:00080000:2.0:1787828825.959222:0:468:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787828825.959228:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828825.959231:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1787828825.959235:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4298300160 02000000:00000001:2.0:1787828825.959236:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828825.959248:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828825.959249:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1787828825.959251:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00000001:1.0:1787828825.959253:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 00000008:00000001:2.0:1787828825.959255:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828825.959260:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828825.959261:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828825.959268:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787828825.959271:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828825.959272:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828825.959273:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787828825.959278:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:1.0:1787828825.959282:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828825.959283:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828825.959295:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828825.959300:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787828825.959306:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2f480 x1874670646768896/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787828825.959318:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787828825.959322:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:1.0:1787828825.959331:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787828825.959334:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787828825.959335:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1787828825.959340:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1787828825.959340:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828825.959343:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787828825.959346:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828825.959347:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828825.959349:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787828825.959357:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787828825.959360:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787828825.959366:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2da40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646768640:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787828825.959369:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787828825.959371:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787828825.959373:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828825.959375:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787828825.959378:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828825.959385:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787828825.959387:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417875008 : -126234291676608 : ffff8d30c9e2da40) 00000100:00000001:1.0:1787828825.959391:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:1.0:1787828825.959404:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828825.959409:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298298902 00000100:00000001:1.0:1787828825.959411:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298300160 00000100:00000001:1.0:1787828825.959413:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298298902 00000100:00000001:1.0:1787828825.959414:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298300160 00000100:00000001:1.0:1787828825.959416:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787828825.959417:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828825.959419:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787828825.959420:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787828825.959421:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828825.959422:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787828825.959423:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787828825.959424:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787828825.959427:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646768768:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787828825.959429:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787828825.959431:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787828825.959432:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828825.959434:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787828825.959435:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828825.959438:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787828825.959439:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417868288 : -126234291683328 : ffff8d30c9e2c000) 00000100:00000001:1.0:1787828825.959442:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:1.0:1787828825.959447:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828825.959450:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298298902 00000100:00000001:1.0:1787828825.959451:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298300160 00000100:00000001:1.0:1787828825.959453:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298298902 00000100:00000001:1.0:1787828825.959455:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298300160 00000100:00000001:1.0:1787828825.959456:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787828825.959457:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828825.959459:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828825.959460:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828825.959467:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787828825.959468:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:1.0:1787828825.959470:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787828825.959471:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1787828825.959472:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787828825.959473:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787828825.959474:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828825.959475:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787828825.959477:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787828825.959478:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787828825.959481:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646768896:192.168.203.143@tcp:8:kworker.0 00000100:00000001:1.0:1787828825.959483:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787828825.959484:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787828825.959485:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828825.959486:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787828825.959508:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828825.959515:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787828825.959516:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417881728 : -126234291669888 : ffff8d30c9e2f480) 00000100:00000001:1.0:1787828825.959519:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:1.0:1787828825.959526:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828825.959530:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298298915 00000100:00000001:1.0:1787828825.959533:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298300160 00000100:00000001:1.0:1787828825.959535:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298298915 00000100:00000001:1.0:1787828825.959537:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298300160 00000100:00000001:1.0:1787828825.959540:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787828825.959542:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828825.959544:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828825.959546:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828825.959549:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787828825.959551:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:1.0:1787828825.959553:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787828825.959555:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1787828825.959556:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828825.959559:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828826.854679:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828826.854684:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828826.854735:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828826.854739:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828826.854741:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828827.878687:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828827.878710:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828827.878761:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828827.878765:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828827.878768:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828828.902691:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828828.902696:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828828.902747:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828828.902750:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828828.902753:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:3.0:1787828829.030860:0:2379:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.143:988 00000400:00000100:3.0:1787828829.030903:0:2379:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.143@tcp at host 192.168.203.143 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:3.0:1787828829.030919:0:2379:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.143@tcp 00000100:00000001:3.0:1787828829.030933:0:2379:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787828829.030953:0:2379:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787828829.030957:0:2379:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828829.030959:0:2379:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:3.0:1787828829.030970:0:2379:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.143@tcp 00000100:00000001:3.0:1787828829.030974:0:2379:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787828829.030977:0:2379:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787828829.030978:0:2379:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828829.030980:0:2379:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:3.0:1787828829.030985:0:2379:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.143@tcp 00000100:00000001:3.0:1787828829.030995:0:2379:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787828829.030999:0:2379:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787828829.031001:0:2379:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828829.031003:0:2379:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:3.0:1787828829.031008:0:2379:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.143@tcp 00000100:00000001:1.0:1787828829.031012:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1787828829.031013:0:2379:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787828829.031019:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787828829.031019:0:2379:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787828829.031023:0:2379:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828829.031025:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00000001:3.0:1787828829.031027:0:2379:0:(events.c:68:request_out_callback()) Process leaving 00000100:00100000:1.0:1787828829.031040:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828825/real 1787828829] req@ffff8d30c9e2f480 x1874670646768896/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828850 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787828829.031057:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787828829.031059:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787828829.031082:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787828829.031085:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787828829.031087:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828829.031091:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787828829.031093:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787828829.031100:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2f480 x1874670646768896/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828850 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787828829.031109:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828829.031113:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787828829.031116:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828829.031118:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787828829.031120:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828829.031123:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787828829.031126:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787828829.031130:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828829.031135:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787828829.031138:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787828829.031143:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646768896:192.168.203.143@tcp:8:kworker.0 00000100:00000001:1.0:1787828829.031149:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787828829.031153:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828825/real 1787828829] req@ffff8d30c9e2da40 x1874670646768640/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828850 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787828829.031163:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787828829.031164:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787828829.031167:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787828829.031169:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787828829.031171:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828829.031173:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787828829.031174:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787828829.031178:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2da40 x1874670646768640/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828850 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787828829.031186:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828829.031188:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787828829.031191:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828829.031193:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787828829.031194:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828829.031196:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787828829.031198:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787828829.031200:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828829.031204:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787828829.031207:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787828829.031210:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2da40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646768640:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787828829.031215:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787828829.031220:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828825/real 1787828829] req@ffff8d30c9e2c000 x1874670646768768/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828850 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787828829.031228:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787828829.031230:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787828829.031232:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787828829.031234:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787828829.031236:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828829.031237:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787828829.031239:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787828829.031243:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2c000 x1874670646768768/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828850 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787828829.031250:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828829.031252:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787828829.031255:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828829.031257:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787828829.031258:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828829.031260:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787828829.031262:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787828829.031265:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828829.031270:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787828829.031272:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787828829.031276:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646768768:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787828829.031280:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787828829.031285:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828825/real 1787828829] req@ffff8d30c9e2ed00 x1874670646768512/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828850 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787828829.031310:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787828829.031311:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787828829.031316:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787828829.031318:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787828829.031320:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828829.031322:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787828829.031323:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787828829.031328:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2ed00 x1874670646768512/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828850 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787828829.031336:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828829.031338:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787828829.031340:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828829.031342:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787828829.031343:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828829.031345:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787828829.031347:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787828829.031350:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828829.031353:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787828829.031356:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787828829.031359:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646768512:192.168.203.143@tcp:250:kworker.0 00000100:00000001:1.0:1787828829.031364:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828829.031366:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787828829.031368:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787828829.031372:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787828829.031379:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787828829.031383:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787828829.031385:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787828829.031390:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787828829.031393:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787828829.031397:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787828829.031399:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828829.031401:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787828829.031403:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787828829.031404:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787828829.031407:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787828829.031408:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787828829.031410:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787828829.031412:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787828829.031414:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787828829.031417:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787828829.031418:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828829.031420:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787828829.031421:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787828829.031422:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787828829.031425:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787828829.031427:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787828829.031428:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787828829.031430:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787828829.031431:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787828829.031434:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787828829.031435:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828829.031437:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787828829.031438:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787828829.031440:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787828829.031442:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787828829.031443:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787828829.031444:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787828829.031446:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787828829.031447:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787828829.031451:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787828829.031452:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828829.031455:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828829.031463:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787828829.031465:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828829.031468:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787828829.031470:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828829.926766:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828829.926777:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828829.926889:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828829.926895:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828829.926899:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828830.950671:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828830.950676:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828830.950775:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828830.950779:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828830.950782:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787828831.078744:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828831.078764:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1787828831.078766:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787828831.078769:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787828831.078774:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787828831.078781:0:40701:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 3631 00000100:00080000:3.0:1787828831.078789:0:40701:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787828831.078792:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787828831.078793:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:3.0:1787828831.078797:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1787828831.078800:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1787828831.078802:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:3.0:1787828831.078804:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:3.0:1787828831.078806:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787828831.078807:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:3.0:1787828831.078811:0:40701:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:3.0:1787828831.078812:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787828831.078815:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787828831.078817:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787828831.078820:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828831.078821:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:3.0:1787828831.078823:0:40701:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787828831.078830:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787828831.078831:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828831.078834:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787828831.078836:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787828831.078840:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828831.078849:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787828831.078857:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c7016d00 x1874670646769024/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787828831.078865:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787828831.078869:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828831.078883:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787828831.078886:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828831.078889:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1787828831.078890:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787828831.078892:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787828831.078894:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787828831.078896:0:40701:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3631 00000100:00080000:3.0:1787828831.078898:0:40701:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787828831.078900:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787828831.078901:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:3.0:1787828831.078903:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1787828831.078904:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1787828831.078906:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000020:00000001:3.0:1787828831.078908:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:3.0:1787828831.078910:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787828831.078911:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:3.0:1787828831.078916:0:40701:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:3.0:1787828831.078919:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787828831.078921:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787828831.078927:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787828831.078930:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828831.078932:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:3.0:1787828831.078938:0:40701:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00000001:1.0:1787828831.078942:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000008:00000001:3.0:1787828831.078943:0:40701:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828831.078949:0:40701:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828831.078962:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 02000000:00000001:3.0:1787828831.078965:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:1.0:1787828831.078967:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 02000000:00000001:3.0:1787828831.078969:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828831.078973:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787828831.078974:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787828831.078978:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:1.0:1787828831.078979:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787828831.078981:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828831.078987:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:3.0:1787828831.078990:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828831.078994:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00100000:3.0:1787828831.078999:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c70170c0 x1874670646769152/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:1.0:1787828831.079001:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787828831.079013:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c7016d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646769024:192.168.203.143@tcp:250:kworker.0 00000100:00000001:3.0:1787828831.079016:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787828831.079020:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828831.079025:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787828831.079028:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1787828831.079033:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00080000:3.0:1787828831.079035:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1787828831.079038:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 02000000:00000001:1.0:1787828831.079040:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000100:00080000:3.0:1787828831.079041:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 02000000:00000001:1.0:1787828831.079047:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828831.079048:0:40701:0:(import.c:485:import_select_connection()) Process entered 02000000:00000001:1.0:1787828831.079054:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00080000:3.0:1787828831.079055:0:40701:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3631 00000100:00080000:3.0:1787828831.079062:0:40701:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 02000000:00000001:1.0:1787828831.079064:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828831.079065:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787828831.079068:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:3.0:1787828831.079072:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1787828831.079073:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1787828831.079076:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000020:00000001:3.0:1787828831.079078:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:3.0:1787828831.079081:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787828831.079082:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:1.0:1787828831.079087:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00080000:3.0:1787828831.079087:0:40701:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:3.0:1787828831.079091:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828831.079093:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369547008 : -126234340004608 : ffff8d30c7016d00) 02000000:00000001:3.0:1787828831.079098:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1787828831.079102:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 02000000:00000001:3.0:1787828831.079104:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787828831.079108:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828831.079111:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:3.0:1787828831.079113:0:40701:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1787828831.079117:0:40701:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828831.079119:0:40701:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787828831.079125:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787828831.079126:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828831.079128:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787828831.079130:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787828831.079133:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828831.079139:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787828831.079149:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c7017840 x1874670646769280/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787828831.079150:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787828831.079165:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787828831.079167:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787828831.079172:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787828831.079172:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828831.079176:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828831.079178:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787828831.079181:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1787828831.079183:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00080000:3.0:1787828831.079189:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1787828831.079191:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:1.0:1787828831.079193:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787828831.079194:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787828831.079200:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:1.0:1787828831.079201:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4298305280 00000100:00080000:3.0:1787828831.079205:0:40701:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3631 00000100:00080000:3.0:1787828831.079211:0:40701:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787828831.079214:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787828831.079216:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:1.0:1787828831.079217:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 00000020:00000001:3.0:1787828831.079218:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1787828831.079222:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1787828831.079225:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000020:00000001:3.0:1787828831.079229:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:3.0:1787828831.079231:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787828831.079233:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:3.0:1787828831.079237:0:40701:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:1.0:1787828831.079240:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4298305280 00000100:00000001:1.0:1787828831.079249:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 00000100:00000001:1.0:1787828831.079254:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787828831.079258:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828831.079262:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00000001:3.0:1787828831.079268:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787828831.079271:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828831/real 1787828831] req@ffff8d30c7016d00 x1874670646769024/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828856 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1787828831.079289:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787828831.079293:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787828831.079296:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828831.079299:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:3.0:1787828831.079301:0:40701:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1787828831.079304:0:40701:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828831.079306:0:40701:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787828831.079311:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787828831.079312:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828831.079315:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787828831.079317:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787828831.079319:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828831.079324:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787828831.079331:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c7014000 x1874670646769408/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787828831.079341:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787828831.079343:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828831.079346:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:1.0:1787828831.079524:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787828831.079526:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787828831.079533:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787828831.079536:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787828831.079538:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828831.079541:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787828831.079543:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787828831.079549:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c7016d00 x1874670646769024/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828856 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787828831.079558:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828831.079562:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787828831.079566:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828831.079569:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787828831.079570:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828831.079573:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787828831.079576:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787828831.079580:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828831.079585:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787828831.079588:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787828831.079592:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c7016d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646769024:192.168.203.143@tcp:250:kworker.0 00000100:00000001:1.0:1787828831.079597:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828831.079623:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787828831.079626:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787828831.079629:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787828831.079633:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787828831.079636:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787828831.079638:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787828831.079644:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787828831.079647:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787828831.079651:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787828831.079652:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828831.079655:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828831.079663:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787828831.079666:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828831.079670:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787828831.079672:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1787828831.079674:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787828831.079676:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787828831.079678:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828831.079681:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787828831.079683:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787828831.079685:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787828831.079690:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c70170c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646769152:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787828831.079694:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787828831.079696:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787828831.079698:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828831.079700:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787828831.079705:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828831.079712:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787828831.079714:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369547968 : -126234340003648 : ffff8d30c70170c0) 00000100:00000001:1.0:1787828831.079719:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1787828831.079738:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787828831.079744:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787828831.079747:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787828831.079749:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828831.079751:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787828831.079754:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828831.079759:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298304023 00000100:00000001:1.0:1787828831.079762:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298305280 00000100:00000001:1.0:1787828831.079765:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298304023 00000100:00000001:1.0:1787828831.079767:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298305280 00000100:00000001:1.0:1787828831.079769:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787828831.079772:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828831.079774:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787828831.079779:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828831/real 1787828831] req@ffff8d30c70170c0 x1874670646769152/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828856 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787828831.079789:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787828831.079790:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787828831.079793:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787828831.079795:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787828831.079796:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828831.079799:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787828831.079800:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787828831.079805:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c70170c0 x1874670646769152/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828856 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787828831.079814:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828831.079817:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787828831.079835:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828831.079837:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787828831.079839:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828831.079841:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787828831.079842:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787828831.079846:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828831.079849:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787828831.079853:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787828831.079856:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c70170c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646769152:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787828831.079860:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787828831.079862:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787828831.079863:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828831.079866:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787828831.079868:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787828831.079871:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787828831.079874:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c7017840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646769280:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787828831.079878:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787828831.079880:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787828831.079882:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828831.079884:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787828831.079887:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828831.079892:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787828831.079893:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369549888 : -126234340001728 : ffff8d30c7017840) 00000100:00000001:1.0:1787828831.079897:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1787828831.079910:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787828831.079915:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787828831.079918:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787828831.079919:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828831.079921:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787828831.079925:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828831.079929:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298304023 00000100:00000001:1.0:1787828831.079932:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298305280 00000100:00000001:1.0:1787828831.079935:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298304023 00000100:00000001:1.0:1787828831.079937:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298305280 00000100:00000001:1.0:1787828831.079940:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787828831.079943:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828831.079945:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787828831.079950:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828831/real 1787828831] req@ffff8d30c7017840 x1874670646769280/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828856 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787828831.079959:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787828831.079960:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787828831.079963:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787828831.079965:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787828831.079966:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828831.079969:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787828831.079970:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787828831.079974:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c7017840 x1874670646769280/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828856 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787828831.079982:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828831.079984:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787828831.079986:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828831.079988:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787828831.079990:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828831.079992:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787828831.079994:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787828831.079996:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828831.080000:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787828831.080002:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787828831.080006:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c7017840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646769280:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787828831.080010:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787828831.080011:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787828831.080013:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828831.080015:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787828831.080017:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787828831.080019:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787828831.080023:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c7014000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646769408:192.168.203.143@tcp:8:kworker.0 00000100:00000001:1.0:1787828831.080026:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787828831.080029:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787828831.080031:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828831.080033:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787828831.080036:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828831.080041:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787828831.080043:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369535488 : -126234340016128 : ffff8d30c7014000) 00000100:00000001:1.0:1787828831.080046:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1787828831.080059:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787828831.080064:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787828831.080067:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787828831.080068:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828831.080070:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787828831.080073:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828831.080077:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298304036 00000100:00000001:1.0:1787828831.080080:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298305280 00000100:00000001:1.0:1787828831.080082:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298304036 00000100:00000001:1.0:1787828831.080084:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298305280 00000100:00000001:1.0:1787828831.080087:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787828831.080089:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828831.080091:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787828831.080095:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828831/real 1787828831] req@ffff8d30c7014000 x1874670646769408/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828856 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787828831.080104:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787828831.080106:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787828831.080108:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787828831.080110:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787828831.080112:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828831.080114:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787828831.080116:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787828831.080121:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c7014000 x1874670646769408/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828856 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787828831.080136:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828831.080138:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787828831.080140:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828831.080142:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787828831.080143:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828831.080146:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787828831.080148:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787828831.080150:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828831.080154:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787828831.080156:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787828831.080160:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c7014000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646769408:192.168.203.143@tcp:8:kworker.0 00000100:00000001:1.0:1787828831.080180:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828831.080182:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787828831.080184:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787828831.080185:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787828831.080188:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787828831.080190:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787828831.080191:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787828831.080194:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787828831.080196:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787828831.080198:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787828831.080200:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828831.080202:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787828831.080203:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787828831.080205:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787828831.080208:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787828831.080209:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787828831.080211:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787828831.080213:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787828831.080215:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787828831.080217:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787828831.080218:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828831.080220:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787828831.080222:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787828831.080223:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787828831.080225:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787828831.080227:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787828831.080228:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787828831.080237:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787828831.080237:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787828831.080239:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787828831.080241:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828831.080243:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828831.080248:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787828831.080250:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828831.080252:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787828831.080254:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828831.974754:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828831.974764:0:815:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1787828831.974779:0:815:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1787828831.974783:0:815:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828831.974789:0:815:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1787828831.974792:0:815:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828831.974794:0:815:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828831.974797:0:815:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828831.974862:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828831.974866:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828831.974869:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828832.998737:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828832.998743:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828832.998860:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828832.998864:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828832.998868:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828834.022690:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828834.022695:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828834.022747:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828834.022752:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828834.022756:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828835.046680:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828835.046686:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828835.046763:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828835.046769:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828835.046775:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828836.070707:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828836.070711:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828836.070778:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828836.070783:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828836.070786:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787828836.198688:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828836.198696:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1787828836.198698:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787828836.198701:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787828836.198705:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787828836.198711:0:468:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 3636 00000100:00080000:2.0:1787828836.198719:0:468:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787828836.198721:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828836.198723:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787828836.198726:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787828836.198728:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787828836.198731:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:2.0:1787828836.198733:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:2.0:1787828836.198734:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828836.198735:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:2.0:1787828836.198738:0:468:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787828836.198740:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828836.198742:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787828836.198744:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828836.198746:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828836.198748:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:2.0:1787828836.198750:0:468:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828836.198760:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787828836.198761:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828836.198763:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787828836.198765:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787828836.198768:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828836.198777:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787828836.198784:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2f0c0 x1874670646769536/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787828836.198797:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787828836.198800:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828836.198812:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787828836.198814:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828836.198816:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1787828836.198817:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787828836.198818:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787828836.198820:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787828836.198821:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3636 00000100:00080000:2.0:1787828836.198824:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787828836.198825:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828836.198826:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787828836.198827:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787828836.198828:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787828836.198830:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000020:00000001:2.0:1787828836.198832:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:2.0:1787828836.198834:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828836.198835:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:2.0:1787828836.198837:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787828836.198839:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828836.198840:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787828836.198844:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828836.198846:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828836.198847:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1787828836.198851:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00000001:1.0:1787828836.198853:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000008:00000001:2.0:1787828836.198853:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828836.198858:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828836.198860:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 02000000:00000001:2.0:1787828836.198861:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:1.0:1787828836.198863:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 02000000:00000001:2.0:1787828836.198864:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828836.198866:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787828836.198868:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:1.0:1787828836.198869:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828836.198873:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1787828836.198874:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787828836.198877:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828836.198878:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787828836.198883:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787828836.198888:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646769536:192.168.203.143@tcp:250:kworker.0 00000100:00000001:2.0:1787828836.198888:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828836.198895:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787828836.198899:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000100:00100000:2.0:1787828836.198900:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2e940 x1874670646769664/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:1.0:1787828836.198902:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828836.198907:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787828836.198912:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828836.198916:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787828836.198921:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828836.198924:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828836.198927:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00080000:2.0:1787828836.198927:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1787828836.198930:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417880768 : -126234291670848 : ffff8d30c9e2f0c0) 00000100:00080000:2.0:1787828836.198932:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:1.0:1787828836.198935:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:2.0:1787828836.198935:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787828836.198938:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787828836.198941:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787828836.198943:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3636 00000100:00080000:2.0:1787828836.198948:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787828836.198949:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828836.198950:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787828836.198951:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787828836.198952:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787828836.198954:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000020:00000001:2.0:1787828836.198956:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:2.0:1787828836.198957:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828836.198958:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:2.0:1787828836.198960:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787828836.198962:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828836.198963:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787828836.198965:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828836.198967:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828836.198969:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1787828836.198970:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1787828836.198973:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828836.198975:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:1.0:1787828836.198977:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 02000000:00000001:2.0:1787828836.198981:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787828836.198983:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828836.198985:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787828836.198987:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:1.0:1787828836.198988:0:2390:0:(events.c:34:request_out_callback()) Process entered 02000000:00000001:2.0:1787828836.198989:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828836.198994:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828836.198996:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828836.198997:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828836.199001:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00100000:2.0:1787828836.199003:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2de00 x1874670646769792/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787828836.199007:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828836.199014:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4298310400 00000100:00000001:2.0:1787828836.199014:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787828836.199019:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828836.199021:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787828836.199023:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1787828836.199025:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 00000100:00080000:2.0:1787828836.199025:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1787828836.199028:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787828836.199030:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787828836.199033:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787828836.199034:0:468:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3636 00000100:00080000:2.0:1787828836.199036:0:468:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787828836.199038:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787828836.199039:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4298310400 00000100:00000001:2.0:1787828836.199039:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787828836.199045:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:1.0:1787828836.199046:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 00000020:00000001:2.0:1787828836.199046:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787828836.199047:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000020:00000001:2.0:1787828836.199052:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:1.0:1787828836.199053:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787828836.199056:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787828836.199058:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828836.199059:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:1.0:1787828836.199062:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00080000:2.0:1787828836.199064:0:468:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787828836.199068:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787828836.199071:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828836/real 1787828836] req@ffff8d30c9e2f0c0 x1874670646769536/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828861 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1787828836.199071:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787828836.199075:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828836.199080:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828836.199084:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1787828836.199086:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000400:00000001:1.0:1787828836.199089:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000008:00000001:2.0:1787828836.199090:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:1.0:1787828836.199093:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1787828836.199094:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828836.199097:0:2390:0:(events.c:78:reply_in_callback()) Process entered 02000000:00000001:2.0:1787828836.199100:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:1.0:1787828836.199101:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787828836.199106:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:2.0:1787828836.199106:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828836.199113:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828836.199113:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:1.0:1787828836.199117:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:2.0:1787828836.199118:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00100000:1.0:1787828836.199123:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2f0c0 x1874670646769536/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828861 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1787828836.199123:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828836.199132:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828836.199133:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828836.199139:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787828836.199142:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828836.199145:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00100000:2.0:1787828836.199145:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2c3c0 x1874670646769920/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787828836.199147:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828836.199152:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787828836.199154:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787828836.199157:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00080000:1.0:1787828836.199158:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00000001:2.0:1787828836.199161:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828836.199165:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1787828836.199166:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787828836.199171:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787828836.199175:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646769536:192.168.203.143@tcp:250:kworker.0 00000100:00000001:1.0:1787828836.199179:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828836.199181:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787828836.199182:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787828836.199184:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787828836.199187:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787828836.199189:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787828836.199190:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787828836.199194:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787828836.199195:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787828836.199197:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787828836.199198:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828836.199200:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828836.199205:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787828836.199206:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828836.199208:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787828836.199209:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1787828836.199210:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787828836.199211:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787828836.199212:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828836.199213:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787828836.199215:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787828836.199216:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787828836.199218:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646769664:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787828836.199220:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787828836.199222:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787828836.199223:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828836.199224:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787828836.199226:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828836.199229:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787828836.199230:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417878848 : -126234291672768 : ffff8d30c9e2e940) 00000100:00000001:1.0:1787828836.199234:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1787828836.199258:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787828836.199262:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787828836.199264:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787828836.199265:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828836.199266:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787828836.199268:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828836.199271:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298309142 00000100:00000001:1.0:1787828836.199273:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298310400 00000100:00000001:1.0:1787828836.199275:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298309142 00000100:00000001:1.0:1787828836.199276:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298310400 00000100:00000001:1.0:1787828836.199278:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787828836.199279:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828836.199280:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787828836.199284:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828836/real 1787828836] req@ffff8d30c9e2e940 x1874670646769664/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828861 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787828836.199290:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787828836.199291:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787828836.199292:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787828836.199293:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787828836.199294:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828836.199295:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787828836.199296:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787828836.199298:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2e940 x1874670646769664/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828861 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787828836.199303:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828836.199305:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787828836.199306:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828836.199307:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787828836.199308:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828836.199309:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787828836.199310:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787828836.199312:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828836.199314:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787828836.199315:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787828836.199317:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646769664:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787828836.199320:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787828836.199321:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787828836.199321:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828836.199323:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787828836.199324:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787828836.199325:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787828836.199327:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646769792:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787828836.199329:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787828836.199331:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787828836.199332:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828836.199333:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787828836.199335:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828836.199338:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787828836.199338:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417875968 : -126234291675648 : ffff8d30c9e2de00) 00000100:00000001:1.0:1787828836.199341:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1787828836.199347:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787828836.199349:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787828836.199350:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787828836.199351:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828836.199352:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787828836.199354:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828836.199356:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298309142 00000100:00000001:1.0:1787828836.199358:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298310400 00000100:00000001:1.0:1787828836.199359:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298309142 00000100:00000001:1.0:1787828836.199361:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298310400 00000100:00000001:1.0:1787828836.199362:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787828836.199363:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828836.199364:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787828836.199366:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828836/real 1787828836] req@ffff8d30c9e2de00 x1874670646769792/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828861 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787828836.199371:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787828836.199372:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787828836.199373:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787828836.199374:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787828836.199375:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828836.199375:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787828836.199376:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787828836.199378:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2de00 x1874670646769792/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828861 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787828836.199388:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828836.199389:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787828836.199391:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828836.199392:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787828836.199392:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828836.199393:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787828836.199394:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787828836.199396:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828836.199398:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787828836.199399:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787828836.199401:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646769792:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787828836.199403:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787828836.199404:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787828836.199405:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828836.199406:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787828836.199407:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787828836.199408:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787828836.199410:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646769920:192.168.203.143@tcp:8:kworker.0 00000100:00000001:1.0:1787828836.199411:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787828836.199412:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787828836.199413:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828836.199414:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787828836.199416:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828836.199419:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787828836.199420:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417869248 : -126234291682368 : ffff8d30c9e2c3c0) 00000100:00000001:1.0:1787828836.199423:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1787828836.199433:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787828836.199437:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787828836.199439:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787828836.199440:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828836.199442:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787828836.199445:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828836.199448:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298309155 00000100:00000001:1.0:1787828836.199450:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298310400 00000100:00000001:1.0:1787828836.199453:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298309155 00000100:00000001:1.0:1787828836.199455:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298310400 00000100:00000001:1.0:1787828836.199457:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787828836.199460:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828836.199462:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787828836.199466:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828836/real 1787828836] req@ffff8d30c9e2c3c0 x1874670646769920/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828861 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787828836.199474:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787828836.199475:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787828836.199477:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787828836.199478:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787828836.199479:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828836.199480:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787828836.199481:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787828836.199484:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2c3c0 x1874670646769920/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828861 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787828836.199488:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828836.199490:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787828836.199492:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828836.199492:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787828836.199493:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828836.199494:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787828836.199495:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787828836.199497:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787828836.199499:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787828836.199500:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787828836.199502:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646769920:192.168.203.143@tcp:8:kworker.0 00000100:00000001:1.0:1787828836.199505:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828836.199506:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787828836.199507:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787828836.199508:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787828836.199509:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787828836.199510:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787828836.199511:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787828836.199513:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787828836.199514:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787828836.199516:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787828836.199517:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828836.199518:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787828836.199519:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787828836.199520:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787828836.199521:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787828836.199522:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787828836.199523:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787828836.199524:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787828836.199525:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787828836.199527:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787828836.199528:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828836.199528:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787828836.199529:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787828836.199530:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787828836.199532:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787828836.199532:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787828836.199533:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787828836.199534:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787828836.199534:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787828836.199536:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787828836.199537:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828836.199538:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828836.199540:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787828836.199541:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828836.199542:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787828836.199543:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828837.094706:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828837.094714:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828837.094777:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828837.094782:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828837.094786:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828838.118704:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828838.118709:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828838.118774:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828838.118779:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828838.118783:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828839.142682:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828839.142688:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828839.142754:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828839.142759:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828839.142763:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828840.166708:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828840.166715:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828840.166778:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828840.166781:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828840.166784:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828841.190708:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828841.190715:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828841.190807:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828841.190812:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828841.190816:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787828841.318784:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828841.318796:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1787828841.318799:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1787828841.318803:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787828841.318807:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1787828841.318815:0:468:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 3641 00000100:00080000:1.0:1787828841.318826:0:468:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1787828841.318829:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787828841.318831:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:1.0:1787828841.318835:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:1.0:1787828841.318838:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:1.0:1787828841.318841:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:1.0:1787828841.318843:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:1.0:1787828841.318844:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787828841.318845:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:1.0:1787828841.318854:0:468:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:1.0:1787828841.318864:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828841.318866:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1787828841.318870:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787828841.318872:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828841.318874:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:1.0:1787828841.318876:0:468:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828841.318891:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1787828841.318892:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828841.318896:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1787828841.318899:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1787828841.318903:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828841.318916:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787828841.318925:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d13ef480 x1874670646770048/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787828841.318934:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787828841.318938:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828841.318953:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1787828841.318956:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828841.318961:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1787828841.318962:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1787828841.318964:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787828841.318966:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1787828841.318969:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3641 00000100:00080000:1.0:1787828841.318972:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1787828841.318973:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787828841.318974:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:1.0:1787828841.318976:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:1.0:1787828841.318986:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:1.0:1787828841.318988:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000020:00000001:1.0:1787828841.318990:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:2.0:1787828841.318991:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787828841.318992:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787828841.318995:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:1.0:1787828841.318998:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787828841.318998:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1787828841.319001:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828841.319002:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1787828841.319008:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787828841.319009:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 02000000:00000001:1.0:1787828841.319011:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828841.319013:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787828841.319014:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:2.0:1787828841.319016:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000008:00100000:1.0:1787828841.319020:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 02000000:00000001:2.0:1787828841.319022:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000008:00000001:1.0:1787828841.319024:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828841.319026:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787828841.319027:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828841.319031:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828841.319033:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1787828841.319035:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828841.319038:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00100000:2.0:1787828841.319038:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d13ef480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646770048:192.168.203.143@tcp:250:kworker.0 02000000:00000001:1.0:1787828841.319039:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1787828841.319043:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828841.319057:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:1.0:1787828841.319058:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828841.319062:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000100:00100000:1.0:1787828841.319066:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d13ef840 x1874670646770176/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:2.0:1787828841.319067:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828841.319072:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1787828841.319078:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828841.319085:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787828841.319086:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828841.319090:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1787828841.319092:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828841.319094:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:1.0:1787828841.319096:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1787828841.319097:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787828841.319099:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:2.0:1787828841.319099:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787828841.319103:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839541351552 : -126234168200064 : ffff8d30d13ef480) 00000100:00080000:1.0:1787828841.319104:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3641 00000100:00080000:1.0:1787828841.319106:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1787828841.319109:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787828841.319112:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:2.0:1787828841.319113:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000020:00000001:1.0:1787828841.319114:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:1.0:1787828841.319116:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:1.0:1787828841.319126:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000020:00000001:1.0:1787828841.319129:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:1.0:1787828841.319130:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787828841.319131:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:1.0:1787828841.319133:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:1.0:1787828841.319134:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828841.319135:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1787828841.319137:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787828841.319138:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828841.319140:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:1.0:1787828841.319141:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1787828841.319143:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828841.319144:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828841.319148:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1787828841.319148:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828841.319150:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1787828841.319151:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1787828841.319152:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828841.319156:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787828841.319159:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d13eed00 x1874670646770304/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787828841.319164:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787828841.319166:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828841.319167:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1787828841.319169:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:2.0:1787828841.319173:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00080000:1.0:1787828841.319174:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1787828841.319178:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1787828841.319179:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787828841.319181:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1787828841.319183:0:468:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3641 00000100:00080000:1.0:1787828841.319187:0:468:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1787828841.319188:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787828841.319189:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:1.0:1787828841.319191:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:1.0:1787828841.319192:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:2.0:1787828841.319192:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000020:00000001:1.0:1787828841.319196:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000020:00000001:1.0:1787828841.319198:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:2.0:1787828841.319199:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787828841.319200:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787828841.319215:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:2.0:1787828841.319219:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1787828841.319220:0:468:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787828841.319224:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787828841.319225:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828841.319228:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1787828841.319230:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787828841.319232:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828841.319233:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828841.319239:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:1.0:1787828841.319241:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00000001:2.0:1787828841.319241:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4298315520 00000008:00000001:1.0:1787828841.319250:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828841.319252:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828841.319253:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 02000000:00000001:1.0:1787828841.319257:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1787828841.319260:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828841.319261:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1787828841.319263:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1787828841.319264:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828841.319268:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828841.319270:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4298315520 00000100:00100000:1.0:1787828841.319272:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d13ede00 x1874670646770432/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787828841.319277:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 00000100:00000001:1.0:1787828841.319280:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787828841.319282:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787828841.319285:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828841.319287:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828841.319289:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828841.319291:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1787828841.319298:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828841/real 1787828841] req@ffff8d30d13ef480 x1874670646770048/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828866 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1787828841.319307:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787828841.319309:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1787828841.319313:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1787828841.319316:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787828841.319318:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828841.319321:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828841.319323:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787828841.319328:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d13ef480 x1874670646770048/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828866 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787828841.319334:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828841.319337:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787828841.319339:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828841.319340:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828841.319341:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828841.319344:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787828841.319345:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787828841.319347:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828841.319350:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:2.0:1787828841.319352:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1787828841.319354:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d13ef480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646770048:192.168.203.143@tcp:250:kworker.0 00000100:00000001:2.0:1787828841.319358:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828841.319360:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828841.319362:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787828841.319364:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1787828841.319367:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787828841.319369:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:2.0:1787828841.319370:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:2.0:1787828841.319375:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787828841.319376:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1787828841.319379:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787828841.319380:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828841.319382:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828841.319389:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1787828841.319392:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828841.319393:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787828841.319394:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1787828841.319395:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1787828841.319396:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787828841.319398:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828841.319399:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787828841.319400:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828841.319402:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787828841.319404:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d13ef840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646770176:192.168.203.143@tcp:38:kworker.0 00000100:00000001:2.0:1787828841.319406:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787828841.319408:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1787828841.319409:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828841.319411:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1787828841.319412:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828841.319417:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787828841.319418:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839541352512 : -126234168199104 : ffff8d30d13ef840) 00000100:00000001:2.0:1787828841.319422:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1787828841.319431:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:2.0:1787828841.319435:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1787828841.319436:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828841.319437:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828841.319438:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1787828841.319440:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828841.319444:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298314262 00000100:00000001:2.0:1787828841.319446:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298315520 00000100:00000001:2.0:1787828841.319448:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298314262 00000100:00000001:2.0:1787828841.319449:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298315520 00000100:00000001:2.0:1787828841.319451:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787828841.319452:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828841.319453:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1787828841.319456:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828841/real 1787828841] req@ffff8d30d13ef840 x1874670646770176/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828866 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1787828841.319461:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787828841.319461:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1787828841.319463:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1787828841.319464:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787828841.319465:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828841.319466:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828841.319467:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787828841.319470:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d13ef840 x1874670646770176/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828866 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787828841.319474:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828841.319476:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787828841.319478:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828841.319479:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828841.319480:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828841.319481:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787828841.319483:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787828841.319484:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828841.319486:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:2.0:1787828841.319488:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1787828841.319490:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d13ef840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646770176:192.168.203.143@tcp:38:kworker.0 00000100:00000001:2.0:1787828841.319492:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1787828841.319493:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787828841.319493:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828841.319495:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787828841.319496:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828841.319497:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787828841.319499:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d13eed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646770304:192.168.203.143@tcp:38:kworker.0 00000100:00000001:2.0:1787828841.319501:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787828841.319502:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1787828841.319503:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828841.319505:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1787828841.319506:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828841.319509:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787828841.319510:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839541349632 : -126234168201984 : ffff8d30d13eed00) 00000100:00000001:2.0:1787828841.319512:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1787828841.319518:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:2.0:1787828841.319520:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1787828841.319521:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828841.319523:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828841.319524:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1787828841.319525:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828841.319527:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298314262 00000100:00000001:2.0:1787828841.319529:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298315520 00000100:00000001:2.0:1787828841.319531:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298314262 00000100:00000001:2.0:1787828841.319533:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298315520 00000100:00000001:2.0:1787828841.319534:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787828841.319536:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828841.319537:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1787828841.319539:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828841/real 1787828841] req@ffff8d30d13eed00 x1874670646770304/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828866 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1787828841.319544:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787828841.319544:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1787828841.319545:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1787828841.319546:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787828841.319547:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828841.319548:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828841.319549:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787828841.319551:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d13eed00 x1874670646770304/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828866 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787828841.319556:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828841.319557:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787828841.319559:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828841.319560:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828841.319561:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828841.319562:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787828841.319563:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787828841.319564:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828841.319566:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:2.0:1787828841.319567:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1787828841.319569:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d13eed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646770304:192.168.203.143@tcp:38:kworker.0 00000100:00000001:2.0:1787828841.319571:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1787828841.319572:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787828841.319573:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828841.319575:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787828841.319576:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828841.319577:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787828841.319579:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d13ede00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646770432:192.168.203.143@tcp:8:kworker.0 00000100:00000001:2.0:1787828841.319592:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787828841.319593:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1787828841.319594:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828841.319595:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1787828841.319596:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828841.319619:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787828841.319620:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839541345792 : -126234168205824 : ffff8d30d13ede00) 00000100:00000001:2.0:1787828841.319622:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1787828841.319629:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:2.0:1787828841.319632:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1787828841.319634:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828841.319635:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828841.319636:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1787828841.319637:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828841.319639:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298314276 00000100:00000001:2.0:1787828841.319641:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298315520 00000100:00000001:2.0:1787828841.319643:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298314276 00000100:00000001:2.0:1787828841.319644:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298315520 00000100:00000001:2.0:1787828841.319646:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787828841.319647:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828841.319648:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1787828841.319651:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828841/real 1787828841] req@ffff8d30d13ede00 x1874670646770432/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828866 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1787828841.319655:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787828841.319656:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1787828841.319657:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1787828841.319658:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787828841.319659:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828841.319661:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828841.319662:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787828841.319665:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d13ede00 x1874670646770432/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828866 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787828841.319670:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828841.319671:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787828841.319672:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828841.319673:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828841.319674:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828841.319675:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787828841.319676:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787828841.319677:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828841.319679:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:2.0:1787828841.319680:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1787828841.319682:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d13ede00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646770432:192.168.203.143@tcp:8:kworker.0 00000100:00000001:2.0:1787828841.319685:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828841.319697:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828841.319698:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787828841.319699:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1787828841.319701:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787828841.319702:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:2.0:1787828841.319703:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:2.0:1787828841.319705:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787828841.319706:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1787828841.319707:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787828841.319708:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828841.319709:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828841.319710:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787828841.319710:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1787828841.319711:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787828841.319712:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:2.0:1787828841.319713:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:2.0:1787828841.319714:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787828841.319715:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1787828841.319716:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787828841.319718:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828841.319719:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828841.319720:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787828841.319722:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1787828841.319724:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787828841.319725:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:2.0:1787828841.319726:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:2.0:1787828841.319728:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787828841.319729:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1787828841.319732:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787828841.319733:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828841.319735:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828841.319738:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1787828841.319740:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828841.319742:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787828841.319744:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828842.214708:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828842.214715:0:815:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1787828842.214717:0:815:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1787828842.214720:0:815:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828842.214724:0:815:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1787828842.214726:0:815:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828842.214727:0:815:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828842.214729:0:815:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828842.214803:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828842.214808:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828842.214814:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828843.238754:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828843.238778:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828843.238824:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828843.238828:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828843.238831:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828844.262703:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828844.262709:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828844.262757:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828844.262760:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828844.262763:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828845.286682:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828845.286687:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828845.286744:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828845.286747:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828845.286750:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828846.310762:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828846.310774:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828846.310864:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828846.310869:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828846.310874:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787828846.438720:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828846.438737:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1787828846.438742:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1787828846.438748:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787828846.438755:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1787828846.438767:0:468:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 3646 00000100:00080000:1.0:1787828846.438785:0:468:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1787828846.438790:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787828846.438793:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:1.0:1787828846.438800:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:1.0:1787828846.438805:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:1.0:1787828846.438810:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:1.0:1787828846.438815:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:1.0:1787828846.438818:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787828846.438821:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:1.0:1787828846.438829:0:468:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:1.0:1787828846.438834:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828846.438838:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1787828846.438844:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787828846.438850:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828846.438854:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:1.0:1787828846.438858:0:468:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828846.438877:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1787828846.438881:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828846.438886:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1787828846.438891:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1787828846.438898:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828846.438944:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787828846.438977:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d13ef0c0 x1874670646770560/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787828846.438992:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787828846.439018:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828846.439038:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1787828846.439043:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828846.439048:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1787828846.439050:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1787828846.439053:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787828846.439057:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1787828846.439079:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3646 00000100:00080000:1.0:1787828846.439085:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1787828846.439090:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787828846.439093:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:1.0:1787828846.439098:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:1.0:1787828846.439100:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:1.0:1787828846.439105:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000020:00000001:1.0:1787828846.439109:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:1.0:1787828846.439122:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828846.439123:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787828846.439125:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:2.0:1787828846.439134:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00080000:1.0:1787828846.439136:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787828846.439143:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787828846.439145:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828846.439150:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 02000000:00000001:1.0:1787828846.439153:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:2.0:1787828846.439158:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828846.439194:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828846.439197:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787828846.439219:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828846.439222:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787828846.439225:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 02000000:00000001:2.0:1787828846.439231:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000008:00100000:1.0:1787828846.439236:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1787828846.439251:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787828846.439254:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d13ef0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646770560:192.168.203.143@tcp:250:kworker.0 00000100:00000001:1.0:1787828846.439256:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828846.439266:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1787828846.439270:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828846.439273:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1787828846.439275:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:2.0:1787828846.439277:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787828846.439279:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828846.439287:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:1.0:1787828846.439289:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828846.439294:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787828846.439303:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d13efc00 x1874670646770688/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:2.0:1787828846.439303:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1787828846.439313:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828846.439318:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787828846.439323:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828846.439329:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1787828846.439334:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828846.439339:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:1.0:1787828846.439341:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:2.0:1787828846.439343:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00080000:1.0:1787828846.439344:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787828846.439349:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839541350592 : -126234168201024 : ffff8d30d13ef0c0) 00000100:00000001:1.0:1787828846.439351:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1787828846.439359:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3646 00000100:00000001:2.0:1787828846.439361:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00080000:1.0:1787828846.439366:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1787828846.439372:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787828846.439374:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:1.0:1787828846.439377:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:1.0:1787828846.439380:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:1.0:1787828846.439382:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000020:00000001:1.0:1787828846.439386:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:1.0:1787828846.439390:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787828846.439393:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:1.0:1787828846.439397:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:1.0:1787828846.439401:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828846.439405:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1787828846.439409:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787828846.439411:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828846.439414:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:1.0:1787828846.439416:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1787828846.439420:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828846.439424:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:2.0:1787828846.439443:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 02000000:00000001:1.0:1787828846.439450:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1787828846.439455:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828846.439458:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1787828846.439461:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1787828846.439465:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828846.439468:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787828846.439473:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828846.439478:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828846.439486:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787828846.439487:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d13ed2c0 x1874670646770816/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787828846.439493:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787828846.439506:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787828846.439508:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828846.439510:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828846.439519:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828846.439522:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4298320640 00000100:00080000:1.0:1787828846.439524:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828846.439534:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1787828846.439537:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1787828846.439540:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787828846.439544:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:2.0:1787828846.439546:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 00000100:00080000:1.0:1787828846.439550:0:468:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3646 00000100:00080000:1.0:1787828846.439560:0:468:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787828846.439563:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4298320640 00000100:00000001:1.0:1787828846.439574:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787828846.439581:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:2.0:1787828846.439582:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 00000020:00000001:1.0:1787828846.439589:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:2.0:1787828846.439593:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000020:00000001:1.0:1787828846.439596:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:2.0:1787828846.439601:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787828846.439603:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:2.0:1787828846.439609:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000020:00000001:1.0:1787828846.439612:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:1.0:1787828846.439618:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787828846.439620:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:1.0:1787828846.439628:0:468:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00100000:2.0:1787828846.439669:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828846/real 1787828846] req@ffff8d30d13ef0c0 x1874670646770560/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828871 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787828846.439689:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:2.0:1787828846.439691:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 02000000:00000001:1.0:1787828846.439697:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000400:00000001:2.0:1787828846.439698:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 02000000:00000001:1.0:1787828846.439706:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787828846.439709:0:2390:0:(events.c:78:reply_in_callback()) Process entered 02000000:00000001:1.0:1787828846.439713:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828846.439717:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787828846.439721:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:2.0:1787828846.439724:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000008:00100000:1.0:1787828846.439728:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00000001:2.0:1787828846.439731:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000008:00000001:1.0:1787828846.439738:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828846.439739:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828846.439748:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787828846.439753:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d13ef0c0 x1874670646770560/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828871 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:1.0:1787828846.439759:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1787828846.439765:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828846.439773:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1787828846.439778:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1787828846.439785:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828846.439799:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828846.439799:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828846.439810:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00100000:1.0:1787828846.439819:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d13ed680 x1874670646770944/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787828846.439820:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828846.439827:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828846.439831:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828846.439839:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787828846.439844:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787828846.439848:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00080000:2.0:1787828846.439852:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00000001:1.0:1787828846.439856:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828846.439865:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787828846.439866:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:2.0:1787828846.439876:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1787828846.439884:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d13ef0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646770560:192.168.203.143@tcp:250:kworker.0 00000100:00000001:2.0:1787828846.439902:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828846.439906:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828846.439910:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787828846.439915:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1787828846.439920:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787828846.439924:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:2.0:1787828846.439927:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:2.0:1787828846.439934:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787828846.439936:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1787828846.439941:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787828846.439943:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828846.439947:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828846.439969:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1787828846.439972:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828846.439976:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787828846.439979:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1787828846.439981:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1787828846.439984:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787828846.439986:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828846.440006:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787828846.440009:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828846.440012:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787828846.440018:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d13efc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646770688:192.168.203.143@tcp:38:kworker.0 00000100:00000001:2.0:1787828846.440023:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787828846.440026:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1787828846.440028:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828846.440031:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1787828846.440035:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828846.440043:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787828846.440045:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839541353472 : -126234168198144 : ffff8d30d13efc00) 00000100:00000001:2.0:1787828846.440052:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1787828846.440095:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:2.0:1787828846.440103:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1787828846.440107:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828846.440110:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828846.440113:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1787828846.440117:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828846.440125:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298319383 00000100:00000001:2.0:1787828846.440129:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298320640 00000100:00000001:2.0:1787828846.440133:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298319383 00000100:00000001:2.0:1787828846.440137:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298320640 00000100:00000001:2.0:1787828846.440140:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787828846.440144:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828846.440147:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1787828846.440155:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828846/real 1787828846] req@ffff8d30d13efc00 x1874670646770688/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828871 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1787828846.440195:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787828846.440197:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1787828846.440218:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1787828846.440220:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787828846.440222:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828846.440225:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828846.440227:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787828846.440233:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d13efc00 x1874670646770688/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828871 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787828846.440251:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828846.440254:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787828846.440258:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828846.440260:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828846.440262:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828846.440265:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787828846.440267:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787828846.440271:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828846.440275:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:2.0:1787828846.440278:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1787828846.440283:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d13efc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646770688:192.168.203.143@tcp:38:kworker.0 00000100:00000001:2.0:1787828846.440288:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1787828846.440290:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787828846.440292:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828846.440295:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787828846.440298:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828846.440301:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787828846.440305:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d13ed2c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646770816:192.168.203.143@tcp:38:kworker.0 00000100:00000001:2.0:1787828846.440309:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787828846.440312:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1787828846.440314:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828846.440317:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1787828846.440322:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828846.440328:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787828846.440330:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839541342912 : -126234168208704 : ffff8d30d13ed2c0) 00000100:00000001:2.0:1787828846.440335:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1787828846.440351:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:2.0:1787828846.440357:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1787828846.440360:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828846.440363:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828846.440366:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1787828846.440369:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828846.440374:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298319383 00000100:00000001:2.0:1787828846.440377:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298320640 00000100:00000001:2.0:1787828846.440380:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298319383 00000100:00000001:2.0:1787828846.440383:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298320640 00000100:00000001:2.0:1787828846.440386:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787828846.440389:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828846.440391:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1787828846.440397:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828846/real 1787828846] req@ffff8d30d13ed2c0 x1874670646770816/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828871 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1787828846.440408:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787828846.440410:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1787828846.440413:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1787828846.440415:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787828846.440433:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828846.440436:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828846.440439:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787828846.440445:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d13ed2c0 x1874670646770816/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828871 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787828846.440455:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828846.440458:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787828846.440462:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828846.440464:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828846.440466:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828846.440470:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787828846.440472:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787828846.440476:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828846.440481:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:2.0:1787828846.440485:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1787828846.440490:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d13ed2c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646770816:192.168.203.143@tcp:38:kworker.0 00000100:00000001:2.0:1787828846.440496:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1787828846.440499:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787828846.440501:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828846.440504:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787828846.440507:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828846.440510:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787828846.440515:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d13ed680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646770944:192.168.203.143@tcp:8:kworker.0 00000100:00000001:2.0:1787828846.440520:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787828846.440523:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1787828846.440525:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828846.440528:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1787828846.440532:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828846.440540:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787828846.440542:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839541343872 : -126234168207744 : ffff8d30d13ed680) 00000100:00000001:2.0:1787828846.440546:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1787828846.440563:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:2.0:1787828846.440570:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1787828846.440573:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828846.440575:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828846.440578:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1787828846.440582:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828846.440587:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298319396 00000100:00000001:2.0:1787828846.440591:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298320640 00000100:00000001:2.0:1787828846.440594:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298319396 00000100:00000001:2.0:1787828846.440598:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298320640 00000100:00000001:2.0:1787828846.440601:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787828846.440605:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828846.440607:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1787828846.440614:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828846/real 1787828846] req@ffff8d30d13ed680 x1874670646770944/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828871 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1787828846.440667:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787828846.440669:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1787828846.440674:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1787828846.440677:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787828846.440679:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828846.440682:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828846.440684:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787828846.440691:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d13ed680 x1874670646770944/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828871 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787828846.440707:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828846.440711:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787828846.440715:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828846.440717:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828846.440719:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828846.440722:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787828846.440725:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787828846.440729:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828846.440734:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:2.0:1787828846.440737:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1787828846.440742:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d13ed680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646770944:192.168.203.143@tcp:8:kworker.0 00000100:00000001:2.0:1787828846.440749:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828846.440752:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828846.440754:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787828846.440757:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1787828846.440760:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787828846.440763:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:2.0:1787828846.440765:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:2.0:1787828846.440769:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787828846.440771:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1787828846.440774:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787828846.440776:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828846.440780:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828846.440781:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787828846.440783:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1787828846.440786:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787828846.440788:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:2.0:1787828846.440790:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:2.0:1787828846.440792:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787828846.440794:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1787828846.440797:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787828846.440799:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828846.440801:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828846.440803:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787828846.440805:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1787828846.440808:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787828846.440810:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:2.0:1787828846.440812:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:2.0:1787828846.440816:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787828846.440818:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1787828846.440820:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787828846.440822:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828846.440826:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828846.440831:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1787828846.440833:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828846.440837:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787828846.440839:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828847.334715:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828847.334722:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828847.334830:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828847.334836:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828847.334841:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828848.358738:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828848.358745:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828848.358824:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828848.358830:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828848.358835:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828849.382737:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828849.382745:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828849.382830:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828849.382835:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828849.382838:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828850.406709:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828850.406716:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828850.406809:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828850.406813:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828850.406816:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828851.430715:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828851.430721:0:815:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1787828851.430723:0:815:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1787828851.430725:0:815:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828851.430729:0:815:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1787828851.430731:0:815:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828851.430733:0:815:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828851.430734:0:815:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828851.430775:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828851.430778:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828851.430781:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787828851.558758:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828851.558767:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1787828851.558769:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787828851.558773:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787828851.558777:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787828851.558782:0:468:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 3652 00000100:00080000:3.0:1787828851.558791:0:468:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787828851.558794:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787828851.558795:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:3.0:1787828851.558799:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1787828851.558801:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1787828851.558804:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:3.0:1787828851.558805:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:3.0:1787828851.558807:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787828851.558808:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:3.0:1787828851.558811:0:468:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:3.0:1787828851.558813:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787828851.558815:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787828851.558818:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787828851.558820:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828851.558822:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:3.0:1787828851.558823:0:468:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787828851.558833:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787828851.558834:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828851.558837:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787828851.558839:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787828851.558842:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828851.558852:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787828851.558861:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c7016940 x1874670646771072/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787828851.558882:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787828851.558886:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828851.558913:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787828851.558917:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828851.558920:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1787828851.558922:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787828851.558924:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787828851.558928:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787828851.558930:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3652 00000100:00080000:3.0:1787828851.558934:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787828851.558936:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787828851.558937:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:3.0:1787828851.558939:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1787828851.558940:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1787828851.558942:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000020:00000001:3.0:1787828851.558943:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:3.0:1787828851.558945:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787828851.558946:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:3.0:1787828851.558949:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:3.0:1787828851.558950:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787828851.558951:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:2.0:1787828851.558955:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:3.0:1787828851.558956:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787828851.558960:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828851.558961:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:2.0:1787828851.558962:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000008:00100000:3.0:1787828851.558965:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00000001:2.0:1787828851.558967:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000008:00000001:3.0:1787828851.558968:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828851.558971:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828851.558972:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 02000000:00000001:3.0:1787828851.558975:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:2.0:1787828851.558976:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787828851.558977:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828851.558983:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787828851.558984:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787828851.558985:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787828851.558986:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828851.558990:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787828851.558992:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828851.558996:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787828851.558999:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c7017c00 x1874670646771200/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787828851.559006:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c7016940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646771072:192.168.203.143@tcp:250:kworker.0 00000100:00000001:3.0:1787828851.559008:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787828851.559012:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828851.559016:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:3.0:1787828851.559016:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787828851.559020:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 02000000:00000001:2.0:1787828851.559023:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000100:00080000:3.0:1787828851.559023:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1787828851.559027:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 02000000:00000001:2.0:1787828851.559028:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787828851.559030:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 02000000:00000001:2.0:1787828851.559033:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:3.0:1787828851.559035:0:468:0:(import.c:485:import_select_connection()) Process entered 02000000:00000001:2.0:1787828851.559040:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787828851.559041:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3652 00000100:00080000:3.0:1787828851.559048:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787828851.559050:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787828851.559052:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:3.0:1787828851.559055:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1787828851.559056:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1787828851.559058:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:2.0:1787828851.559059:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000020:00000001:3.0:1787828851.559061:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:2.0:1787828851.559063:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369546048 : -126234340005568 : ffff8d30c7016940) 00000100:00000001:3.0:1787828851.559066:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787828851.559068:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:3.0:1787828851.559071:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:3.0:1787828851.559072:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787828851.559074:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787828851.559076:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787828851.559080:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828851.559082:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:3.0:1787828851.559083:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1787828851.559086:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828851.559087:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:3.0:1787828851.559088:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787828851.559105:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787828851.559106:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828851.559108:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787828851.559109:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787828851.559110:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828851.559116:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787828851.559119:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c7014000 x1874670646771328/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787828851.559125:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787828851.559127:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828851.559142:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787828851.559144:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828851.559147:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1787828851.559148:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787828851.559149:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787828851.559151:0:468:0:(import.c:485:import_select_connection()) Process entered 00000800:00000100:2.0:1787828851.559155:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00080000:3.0:1787828851.559156:0:468:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3652 00000100:00080000:3.0:1787828851.559161:0:468:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787828851.559163:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787828851.559165:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:3.0:1787828851.559166:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1787828851.559168:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:2.0:1787828851.559169:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000020:00000001:3.0:1787828851.559170:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:2.0:1787828851.559175:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000020:00000001:3.0:1787828851.559175:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:3.0:1787828851.559178:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828851.559179:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828851.559183:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1787828851.559183:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:2.0:1787828851.559190:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787828851.559190:0:468:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:3.0:1787828851.559196:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828851.559197:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4298325760 02000000:00000001:3.0:1787828851.559201:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787828851.559207:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787828851.559208:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 02000000:00000001:3.0:1787828851.559210:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828851.559214:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:3.0:1787828851.559217:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1787828851.559220:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828851.559221:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828851.559223:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4298325760 02000000:00000001:3.0:1787828851.559226:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787828851.559229:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828851.559231:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:2.0:1787828851.559232:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 02000000:00000001:3.0:1787828851.559233:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787828851.559236:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828851.559241:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787828851.559243:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828851.559246:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787828851.559249:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c7017840 x1874670646771456/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787828851.559252:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00000001:3.0:1787828851.559259:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787828851.559263:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787828851.559265:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828851/real 1787828851] req@ffff8d30c7016940 x1874670646771072/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828876 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787828851.559265:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:2.0:1787828851.559279:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787828851.559282:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1787828851.559287:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1787828851.559290:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787828851.559293:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828851.559295:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828851.559297:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787828851.559302:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c7016940 x1874670646771072/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828876 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787828851.559310:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828851.559314:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787828851.559317:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828851.559319:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828851.559320:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828851.559322:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787828851.559324:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787828851.559328:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828851.559332:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:2.0:1787828851.559334:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1787828851.559337:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c7016940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646771072:192.168.203.143@tcp:250:kworker.0 00000100:00000001:2.0:1787828851.559341:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828851.559343:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828851.559344:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787828851.559346:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1787828851.559348:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787828851.559350:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:2.0:1787828851.559352:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:2.0:1787828851.559355:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787828851.559356:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1787828851.559358:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787828851.559359:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828851.559361:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828851.559366:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1787828851.559367:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828851.559369:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787828851.559370:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1787828851.559371:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1787828851.559373:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787828851.559374:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828851.559375:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787828851.559377:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828851.559378:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787828851.559380:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c7017c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646771200:192.168.203.143@tcp:38:kworker.0 00000100:00000001:2.0:1787828851.559382:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787828851.559384:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1787828851.559385:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828851.559386:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1787828851.559388:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828851.559393:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787828851.559394:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369550848 : -126234340000768 : ffff8d30c7017c00) 00000100:00000001:2.0:1787828851.559397:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1787828851.559408:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:2.0:1787828851.559411:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1787828851.559413:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828851.559414:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828851.559415:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1787828851.559417:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828851.559420:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298324502 00000100:00000001:2.0:1787828851.559422:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298325760 00000100:00000001:2.0:1787828851.559424:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298324502 00000100:00000001:2.0:1787828851.559425:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298325760 00000100:00000001:2.0:1787828851.559427:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787828851.559428:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828851.559429:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1787828851.559432:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828851/real 1787828851] req@ffff8d30c7017c00 x1874670646771200/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828876 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1787828851.559437:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787828851.559438:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1787828851.559439:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1787828851.559440:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787828851.559442:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828851.559443:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828851.559444:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787828851.559446:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c7017c00 x1874670646771200/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828876 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787828851.559451:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828851.559452:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787828851.559453:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828851.559454:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828851.559455:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828851.559456:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787828851.559465:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787828851.559467:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828851.559469:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:2.0:1787828851.559470:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1787828851.559472:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c7017c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646771200:192.168.203.143@tcp:38:kworker.0 00000100:00000001:2.0:1787828851.559474:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1787828851.559475:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787828851.559476:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828851.559477:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787828851.559478:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828851.559480:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787828851.559482:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c7014000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646771328:192.168.203.143@tcp:38:kworker.0 00000100:00000001:2.0:1787828851.559483:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787828851.559484:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1787828851.559486:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828851.559487:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1787828851.559489:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828851.559491:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787828851.559492:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369535488 : -126234340016128 : ffff8d30c7014000) 00000100:00000001:2.0:1787828851.559495:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1787828851.559501:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:2.0:1787828851.559503:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1787828851.559504:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828851.559505:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828851.559506:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1787828851.559508:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828851.559510:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298324502 00000100:00000001:2.0:1787828851.559511:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298325760 00000100:00000001:2.0:1787828851.559513:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298324502 00000100:00000001:2.0:1787828851.559515:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298325760 00000100:00000001:2.0:1787828851.559516:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787828851.559517:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828851.559518:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1787828851.559521:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828851/real 1787828851] req@ffff8d30c7014000 x1874670646771328/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828876 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1787828851.559525:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787828851.559526:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1787828851.559527:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1787828851.559528:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787828851.559529:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828851.559530:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828851.559530:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787828851.559533:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c7014000 x1874670646771328/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828876 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787828851.559537:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828851.559538:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787828851.559539:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828851.559540:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828851.559542:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828851.559543:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787828851.559544:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787828851.559545:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828851.559548:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:2.0:1787828851.559549:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1787828851.559551:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c7014000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646771328:192.168.203.143@tcp:38:kworker.0 00000100:00000001:2.0:1787828851.559553:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1787828851.559554:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787828851.559555:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828851.559556:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787828851.559557:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828851.559558:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787828851.559560:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c7017840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646771456:192.168.203.143@tcp:8:kworker.0 00000100:00000001:2.0:1787828851.559562:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787828851.559563:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1787828851.559564:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828851.559565:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1787828851.559566:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828851.559569:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787828851.559570:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369549888 : -126234340001728 : ffff8d30c7017840) 00000100:00000001:2.0:1787828851.559572:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1787828851.559577:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:2.0:1787828851.559579:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1787828851.559581:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828851.559582:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828851.559583:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1787828851.559585:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828851.559587:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298324515 00000100:00000001:2.0:1787828851.559588:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298325760 00000100:00000001:2.0:1787828851.559590:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298324515 00000100:00000001:2.0:1787828851.559591:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298325760 00000100:00000001:2.0:1787828851.559593:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787828851.559594:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828851.559595:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1787828851.559598:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828851/real 1787828851] req@ffff8d30c7017840 x1874670646771456/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828876 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1787828851.559635:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787828851.559636:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1787828851.559638:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1787828851.559639:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787828851.559640:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828851.559641:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828851.559643:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787828851.559645:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c7017840 x1874670646771456/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828876 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787828851.559650:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828851.559652:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787828851.559653:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828851.559654:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828851.559655:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828851.559656:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787828851.559657:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787828851.559658:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828851.559660:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:2.0:1787828851.559661:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1787828851.559663:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c7017840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646771456:192.168.203.143@tcp:8:kworker.0 00000100:00000001:2.0:1787828851.559665:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828851.559666:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828851.559667:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787828851.559668:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1787828851.559670:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787828851.559671:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:2.0:1787828851.559672:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:2.0:1787828851.559673:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787828851.559674:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1787828851.559675:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787828851.559677:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828851.559678:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828851.559679:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787828851.559680:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1787828851.559681:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787828851.559682:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:2.0:1787828851.559683:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:2.0:1787828851.559684:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787828851.559684:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1787828851.559685:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787828851.559686:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828851.559687:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828851.559688:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787828851.559688:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1787828851.559689:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787828851.559690:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:2.0:1787828851.559691:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:2.0:1787828851.559691:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787828851.559692:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1787828851.559693:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787828851.559694:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828851.559695:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828851.559697:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1787828851.559698:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828851.559699:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787828851.559700:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828852.454659:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828852.454664:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828852.454735:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828852.454738:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828852.454741:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828853.478742:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828853.478747:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828853.478894:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828853.478966:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828853.478968:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828854.502710:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828854.502719:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828854.502836:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828854.502840:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828854.502844:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828855.526710:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828855.526715:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828855.526769:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828855.526772:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828855.526775:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828856.550673:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828856.550679:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828856.550732:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828856.550736:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828856.550739:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787828856.678711:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828856.678721:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1787828856.678724:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787828856.678728:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787828856.678733:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787828856.678740:0:40701:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 3657 00000100:00080000:3.0:1787828856.678766:0:40701:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787828856.678768:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787828856.678770:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:3.0:1787828856.678773:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1787828856.678776:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1787828856.678779:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:3.0:1787828856.678781:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:3.0:1787828856.678782:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787828856.678783:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:3.0:1787828856.678786:0:40701:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:3.0:1787828856.678788:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787828856.678790:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787828856.678793:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787828856.678795:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828856.678797:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:3.0:1787828856.678799:0:40701:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787828856.678808:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787828856.678810:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828856.678812:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787828856.678814:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787828856.678818:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828856.678828:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787828856.678835:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c70170c0 x1874670646771584/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787828856.678843:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787828856.678846:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828856.678860:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787828856.678862:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828856.678865:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1787828856.678866:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787828856.678867:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787828856.678869:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787828856.678871:0:40701:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3657 00000100:00080000:3.0:1787828856.678873:0:40701:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787828856.678875:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787828856.678876:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:3.0:1787828856.678879:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1787828856.678880:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1787828856.678882:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000020:00000001:3.0:1787828856.678884:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:3.0:1787828856.678885:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787828856.678887:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:3.0:1787828856.678889:0:40701:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:3.0:1787828856.678890:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828856.678892:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:3.0:1787828856.678892:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787828856.678897:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787828856.678898:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 02000000:00000001:3.0:1787828856.678902:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828856.678903:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787828856.678905:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:2.0:1787828856.678907:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787828856.678910:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000008:00100000:3.0:1787828856.678910:0:40701:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1787828856.678916:0:40701:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828856.678919:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:3.0:1787828856.678919:0:40701:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828856.678922:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787828856.678924:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787828856.678927:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787828856.678927:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828856.678929:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787828856.678931:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787828856.678934:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787828856.678935:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c70170c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646771584:192.168.203.143@tcp:250:kworker.0 00000100:00000001:3.0:1787828856.678938:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828856.678941:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00100000:3.0:1787828856.678945:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c7016d00 x1874670646771712/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:2.0:1787828856.678946:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1787828856.678950:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828856.678954:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:3.0:1787828856.678955:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787828856.678958:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828856.678960:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828856.678961:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787828856.678966:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828856.678969:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1787828856.678971:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787828856.678972:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787828856.678973:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787828856.678975:0:40701:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3657 00000100:00000001:2.0:1787828856.678977:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00080000:3.0:1787828856.678978:0:40701:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787828856.678981:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369547968 : -126234340003648 : ffff8d30c70170c0) 00000100:00000001:3.0:1787828856.678982:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787828856.678985:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:2.0:1787828856.678987:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000020:00000001:3.0:1787828856.678988:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1787828856.678991:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1787828856.678992:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000020:00000001:3.0:1787828856.678994:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:3.0:1787828856.678996:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787828856.678997:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:3.0:1787828856.678999:0:40701:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:3.0:1787828856.679001:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787828856.679002:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787828856.679004:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787828856.679006:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828856.679007:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:3.0:1787828856.679008:0:40701:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1787828856.679010:0:40701:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828856.679012:0:40701:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787828856.679015:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787828856.679016:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828856.679017:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787828856.679017:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787828856.679019:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828856.679022:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:2.0:1787828856.679027:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00100000:3.0:1787828856.679029:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c7014b40 x1874670646771840/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787828856.679035:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787828856.679037:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828856.679039:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787828856.679041:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:2.0:1787828856.679043:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00080000:3.0:1787828856.679044:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1787828856.679047:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787828856.679049:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787828856.679051:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:2.0:1787828856.679052:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00080000:3.0:1787828856.679053:0:40701:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3657 00000100:00000001:2.0:1787828856.679057:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787828856.679058:0:40701:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787828856.679061:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828856.679065:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1787828856.679065:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:3.0:1787828856.679068:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1787828856.679070:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1787828856.679071:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:2.0:1787828856.679073:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828856.679073:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:3.0:1787828856.679074:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787828856.679077:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:2.0:1787828856.679081:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4298330880 00000100:00080000:3.0:1787828856.679081:0:40701:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:3.0:1787828856.679087:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787828856.679088:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787828856.679090:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787828856.679091:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828856.679093:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:3.0:1787828856.679106:0:40701:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00000001:2.0:1787828856.679108:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 00000008:00000001:3.0:1787828856.679109:0:40701:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828856.679113:0:40701:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828856.679118:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4298330880 02000000:00000001:3.0:1787828856.679118:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787828856.679121:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828856.679123:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787828856.679124:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787828856.679126:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828856.679127:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 00000100:00000001:3.0:1787828856.679131:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828856.679132:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787828856.679153:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787828856.679156:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c70152c0 x1874670646771968/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787828856.679157:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00000001:3.0:1787828856.679165:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00100000:2.0:1787828856.679166:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828856/real 1787828856] req@ffff8d30c70170c0 x1874670646771584/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828881 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787828856.679168:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828856.679173:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:2.0:1787828856.679182:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787828856.679184:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1787828856.679187:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1787828856.679190:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787828856.679192:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828856.679193:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828856.679195:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787828856.679199:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c70170c0 x1874670646771584/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828881 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787828856.679205:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828856.679209:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787828856.679211:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828856.679212:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828856.679213:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828856.679215:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787828856.679216:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787828856.679218:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828856.679221:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:2.0:1787828856.679223:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1787828856.679225:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c70170c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646771584:192.168.203.143@tcp:250:kworker.0 00000100:00000001:2.0:1787828856.679229:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828856.679230:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828856.679231:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787828856.679233:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1787828856.679235:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787828856.679237:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:2.0:1787828856.679238:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:2.0:1787828856.679241:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787828856.679242:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1787828856.679244:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787828856.679244:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828856.679246:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828856.679252:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1787828856.679253:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828856.679255:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787828856.679256:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1787828856.679257:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1787828856.679258:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787828856.679259:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828856.679260:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787828856.679262:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828856.679263:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787828856.679266:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c7016d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646771712:192.168.203.143@tcp:38:kworker.0 00000100:00000001:2.0:1787828856.679268:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787828856.679269:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1787828856.679271:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828856.679272:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1787828856.679274:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828856.679277:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787828856.679278:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369547008 : -126234340004608 : ffff8d30c7016d00) 00000100:00000001:2.0:1787828856.679282:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1787828856.679292:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:2.0:1787828856.679295:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1787828856.679296:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828856.679297:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828856.679299:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1787828856.679301:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828856.679304:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298329622 00000100:00000001:2.0:1787828856.679306:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298330880 00000100:00000001:2.0:1787828856.679308:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298329622 00000100:00000001:2.0:1787828856.679309:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298330880 00000100:00000001:2.0:1787828856.679311:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787828856.679312:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828856.679313:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1787828856.679316:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828856/real 1787828856] req@ffff8d30c7016d00 x1874670646771712/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828881 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1787828856.679321:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787828856.679322:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1787828856.679323:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1787828856.679324:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787828856.679325:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828856.679338:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828856.679339:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787828856.679341:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c7016d00 x1874670646771712/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828881 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787828856.679350:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828856.679352:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787828856.679354:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828856.679355:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828856.679355:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828856.679356:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787828856.679365:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787828856.679367:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828856.679369:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:2.0:1787828856.679370:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1787828856.679372:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c7016d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646771712:192.168.203.143@tcp:38:kworker.0 00000100:00000001:2.0:1787828856.679375:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1787828856.679376:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787828856.679377:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828856.679378:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787828856.679380:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828856.679381:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787828856.679383:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c7014b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646771840:192.168.203.143@tcp:38:kworker.0 00000100:00000001:2.0:1787828856.679385:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787828856.679387:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1787828856.679388:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828856.679389:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1787828856.679391:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828856.679394:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787828856.679394:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369538368 : -126234340013248 : ffff8d30c7014b40) 00000100:00000001:2.0:1787828856.679397:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1787828856.679404:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:2.0:1787828856.679406:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1787828856.679408:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828856.679409:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828856.679410:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1787828856.679412:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828856.679414:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298329622 00000100:00000001:2.0:1787828856.679416:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298330880 00000100:00000001:2.0:1787828856.679417:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298329622 00000100:00000001:2.0:1787828856.679419:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298330880 00000100:00000001:2.0:1787828856.679420:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787828856.679421:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828856.679422:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1787828856.679425:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828856/real 1787828856] req@ffff8d30c7014b40 x1874670646771840/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828881 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1787828856.679429:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787828856.679430:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1787828856.679431:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1787828856.679432:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787828856.679433:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828856.679434:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828856.679435:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787828856.679437:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c7014b40 x1874670646771840/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828881 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787828856.679442:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828856.679443:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787828856.679445:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828856.679446:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828856.679447:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828856.679448:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787828856.679449:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787828856.679450:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828856.679452:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:2.0:1787828856.679453:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1787828856.679456:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c7014b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646771840:192.168.203.143@tcp:38:kworker.0 00000100:00000001:2.0:1787828856.679458:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1787828856.679459:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787828856.679460:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828856.679461:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787828856.679462:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828856.679464:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787828856.679465:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c70152c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646771968:192.168.203.143@tcp:8:kworker.0 00000100:00000001:2.0:1787828856.679467:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787828856.679468:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1787828856.679469:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828856.679470:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1787828856.679471:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828856.679474:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787828856.679475:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369540288 : -126234340011328 : ffff8d30c70152c0) 00000100:00000001:2.0:1787828856.679477:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1787828856.679483:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:2.0:1787828856.679485:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1787828856.679486:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828856.679487:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828856.679488:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1787828856.679490:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828856.679492:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298329635 00000100:00000001:2.0:1787828856.679493:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298330880 00000100:00000001:2.0:1787828856.679495:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298329635 00000100:00000001:2.0:1787828856.679496:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298330880 00000100:00000001:2.0:1787828856.679498:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787828856.679499:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828856.679500:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1787828856.679503:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828856/real 1787828856] req@ffff8d30c70152c0 x1874670646771968/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828881 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1787828856.679512:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787828856.679513:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1787828856.679514:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1787828856.679515:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787828856.679516:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828856.679517:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828856.679517:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787828856.679520:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c70152c0 x1874670646771968/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828881 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787828856.679524:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828856.679525:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787828856.679526:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828856.679527:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828856.679528:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828856.679529:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787828856.679530:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787828856.679531:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828856.679533:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:2.0:1787828856.679534:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1787828856.679536:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c70152c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646771968:192.168.203.143@tcp:8:kworker.0 00000100:00000001:2.0:1787828856.679538:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828856.679539:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828856.679540:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787828856.679541:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1787828856.679542:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787828856.679543:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:2.0:1787828856.679544:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:2.0:1787828856.679546:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787828856.679546:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1787828856.679548:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787828856.679549:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828856.679550:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828856.679551:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787828856.679552:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1787828856.679553:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787828856.679554:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:2.0:1787828856.679555:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:2.0:1787828856.679556:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787828856.679557:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1787828856.679558:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787828856.679558:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828856.679559:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828856.679560:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787828856.679561:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1787828856.679562:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787828856.679562:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:2.0:1787828856.679563:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:2.0:1787828856.679564:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787828856.679564:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1787828856.679566:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787828856.679566:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828856.679567:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828856.679569:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1787828856.679570:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828856.679571:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787828856.679572:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828857.574681:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828857.574687:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828857.574751:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828857.574755:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828857.574757:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828858.598702:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828858.598708:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828858.598780:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828858.598787:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828858.598792:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828859.622702:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828859.622708:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828859.622772:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828859.622776:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828859.622779:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828860.646760:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828860.646767:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828860.646834:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828860.646841:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828860.646846:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828861.670754:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828861.670765:0:815:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1787828861.670768:0:815:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1787828861.670772:0:815:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828861.670815:0:815:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1787828861.670817:0:815:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828861.670819:0:815:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828861.670822:0:815:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828861.670967:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828861.670973:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828861.670978:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787828861.798671:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828861.798682:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1787828861.798685:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787828861.798689:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787828861.798694:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787828861.798702:0:40701:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 3662 00000100:00080000:3.0:1787828861.798713:0:40701:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787828861.798717:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787828861.798719:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:3.0:1787828861.798723:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1787828861.798726:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1787828861.798729:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:3.0:1787828861.798731:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:3.0:1787828861.798733:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787828861.798734:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:3.0:1787828861.798740:0:40701:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:3.0:1787828861.798744:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787828861.798749:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787828861.798753:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787828861.798757:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828861.798760:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:3.0:1787828861.798764:0:40701:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787828861.798776:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787828861.798779:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828861.798783:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787828861.798787:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787828861.798792:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828861.798804:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787828861.798815:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c7015680 x1874670646772096/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787828861.798825:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787828861.798829:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828861.798848:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787828861.798851:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828861.798854:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1787828861.798855:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787828861.798856:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787828861.798859:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787828861.798862:0:40701:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3662 00000100:00080000:3.0:1787828861.798867:0:40701:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787828861.798869:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787828861.798870:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:3.0:1787828861.798872:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:0.0:1787828861.798873:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:3.0:1787828861.798874:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1787828861.798879:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:0.0:1787828861.798881:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000020:00000001:3.0:1787828861.798882:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:0.0:1787828861.798886:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787828861.798886:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787828861.798890:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:0.0:1787828861.798891:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787828861.798895:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787828861.798896:0:40701:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:3.0:1787828861.798901:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828861.798902:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787828861.798903:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787828861.798906:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787828861.798911:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787828861.798911:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787828861.798916:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828861.798919:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00100000:0.0:1787828861.798921:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c7015680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646772096:192.168.203.143@tcp:250:kworker.0 00000008:00100000:3.0:1787828861.798925:0:40701:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00000001:0.0:1787828861.798926:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000008:00000001:3.0:1787828861.798930:0:40701:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828861.798932:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:3.0:1787828861.798932:0:40701:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828861.798937:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787828861.798938:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787828861.798941:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828861.798942:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:3.0:1787828861.798943:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787828861.798946:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787828861.798947:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828861.798949:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828861.798953:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787828861.798960:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c70143c0 x1874670646772224/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787828861.798966:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787828861.798967:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787828861.798968:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828861.798971:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369541248 : -126234340010368 : ffff8d30c7015680) 00000100:00000001:0.0:1787828861.798976:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1787828861.799020:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:0.0:1787828861.799031:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1787828861.799035:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787828861.799036:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828861.799037:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787828861.799041:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1787828861.799042:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00080000:3.0:1787828861.799047:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1787828861.799048:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787828861.799050:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787828861.799051:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828861.799052:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787828861.799057:0:40701:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3662 00000100:00080000:3.0:1787828861.799072:0:40701:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787828861.799073:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4298336000 00000100:00000001:3.0:1787828861.799074:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787828861.799078:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:3.0:1787828861.799080:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1787828861.799081:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1787828861.799083:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:0.0:1787828861.799086:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 00000020:00000001:3.0:1787828861.799086:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:3.0:1787828861.799092:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787828861.799094:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:0.0:1787828861.799095:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4298336000 00000100:00080000:3.0:1787828861.799098:0:40701:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:3.0:1787828861.799101:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828861.799102:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 02000000:00000001:3.0:1787828861.799103:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:0.0:1787828861.799107:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787828861.799109:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828861.799112:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787828861.799118:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828861/real 1787828861] req@ffff8d30c7015680 x1874670646772096/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828886 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787828861.799131:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787828861.799134:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787828861.799138:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787828861.799141:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787828861.799143:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828861.799146:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828861.799148:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787828861.799154:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c7015680 x1874670646772096/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828886 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787828861.799163:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828861.799167:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787828861.799170:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828861.799171:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828861.799173:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828861.799175:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787828861.799176:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787828861.799179:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828861.799182:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787828861.799184:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787828861.799187:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c7015680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646772096:192.168.203.143@tcp:250:kworker.0 00000100:00000001:0.0:1787828861.799191:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828861.799194:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828861.799196:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787828861.799198:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787828861.799202:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787828861.799204:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787828861.799206:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787828861.799210:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787828861.799212:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787828861.799216:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 02000000:00000001:3.0:1787828861.799217:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787828861.799218:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828861.799223:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:3.0:1787828861.799223:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828861.799228:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:3.0:1787828861.799231:0:40701:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1787828861.799233:0:40701:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828861.799235:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787828861.799235:0:40701:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787828861.799241:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787828861.799243:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828861.799245:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:0.0:1787828861.799248:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828861.799252:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787828861.799254:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787828861.799255:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1787828861.799256:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787828861.799257:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828861.799259:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787828861.799260:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787828861.799263:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828861.799265:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c70143c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646772224:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787828861.799268:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787828861.799270:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787828861.799272:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828861.799274:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1787828861.799277:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787828861.799280:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:0.0:1787828861.799283:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 02000000:00000001:3.0:1787828861.799284:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828861.799286:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369536448 : -126234340015168 : ffff8d30c70143c0) 00000100:00000001:0.0:1787828861.799292:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:3.0:1787828861.799292:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787828861.799301:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c7014f00 x1874670646772352/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787828861.799307:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787828861.799309:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:0.0:1787828861.799312:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:3.0:1787828861.799316:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828861.799319:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00080000:3.0:1787828861.799320:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1787828861.799323:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00080000:3.0:1787828861.799324:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1787828861.799327:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828861.799328:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:0.0:1787828861.799332:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00080000:3.0:1787828861.799333:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787828861.799336:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828861.799337:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787828861.799342:0:40701:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3662 00000100:00000001:0.0:1787828861.799343:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298334742 00000100:00080000:3.0:1787828861.799345:0:40701:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787828861.799348:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298336000 00000100:00000001:3.0:1787828861.799349:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787828861.799352:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:0.0:1787828861.799353:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298334742 00000020:00000001:3.0:1787828861.799357:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:0.0:1787828861.799358:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298336000 00000020:00000001:3.0:1787828861.799359:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1787828861.799362:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:0.0:1787828861.799363:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000020:00000001:3.0:1787828861.799365:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:0.0:1787828861.799366:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828861.799368:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787828861.799369:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00000001:3.0:1787828861.799371:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:3.0:1787828861.799377:0:40701:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00100000:0.0:1787828861.799378:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828861/real 1787828861] req@ffff8d30c70143c0 x1874670646772224/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828886 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787828861.799379:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787828861.799383:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787828861.799387:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787828861.799391:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:0.0:1787828861.799393:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000001:3.0:1787828861.799394:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000400:00000001:0.0:1787828861.799396:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000008:00100000:3.0:1787828861.799398:0:40701:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00000001:0.0:1787828861.799400:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000008:00000001:3.0:1787828861.799401:0:40701:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828861.799404:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787828861.799405:0:40701:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828861.799408:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:3.0:1787828861.799411:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:0.0:1787828861.799412:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 02000000:00000001:3.0:1787828861.799413:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828861.799414:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787828861.799418:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787828861.799421:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00100000:0.0:1787828861.799423:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c70143c0 x1874670646772224/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828886 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1787828861.799423:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828861.799431:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828861.799432:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828861.799436:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00100000:3.0:1787828861.799438:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c7015a40 x1874670646772480/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787828861.799439:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828861.799444:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828861.799446:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828861.799450:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787828861.799452:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787828861.799455:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00080000:0.0:1787828861.799457:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00000001:3.0:1787828861.799458:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828861.799462:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787828861.799464:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787828861.799468:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787828861.799471:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c70143c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646772224:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787828861.799474:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828861.799475:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828861.799477:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787828861.799479:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787828861.799481:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787828861.799482:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787828861.799483:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787828861.799492:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787828861.799493:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787828861.799494:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787828861.799495:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828861.799496:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828861.799499:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787828861.799500:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828861.799501:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787828861.799502:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787828861.799503:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1787828861.799504:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787828861.799505:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828861.799507:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787828861.799508:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787828861.799509:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828861.799511:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c7014f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646772352:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787828861.799514:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787828861.799515:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787828861.799517:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828861.799518:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1787828861.799521:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828861.799527:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787828861.799528:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369539328 : -126234340012288 : ffff8d30c7014f00) 00000100:00000001:0.0:1787828861.799532:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1787828861.799544:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:0.0:1787828861.799549:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1787828861.799552:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828861.799553:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828861.799555:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787828861.799558:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828861.799562:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298334742 00000100:00000001:0.0:1787828861.799565:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298336000 00000100:00000001:0.0:1787828861.799568:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298334742 00000100:00000001:0.0:1787828861.799571:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298336000 00000100:00000001:0.0:1787828861.799573:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787828861.799575:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828861.799576:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787828861.799580:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828861/real 1787828861] req@ffff8d30c7014f00 x1874670646772352/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828886 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787828861.799589:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787828861.799591:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787828861.799594:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787828861.799596:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787828861.799597:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828861.799631:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828861.799633:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787828861.799638:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c7014f00 x1874670646772352/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828886 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787828861.799644:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828861.799646:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787828861.799648:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828861.799650:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828861.799651:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828861.799653:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787828861.799655:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787828861.799657:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828861.799659:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787828861.799661:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787828861.799663:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c7014f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646772352:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787828861.799665:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1787828861.799666:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787828861.799668:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828861.799669:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787828861.799671:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787828861.799673:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828861.799675:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c7015a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646772480:192.168.203.143@tcp:8:kworker.0 00000100:00000001:0.0:1787828861.799677:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787828861.799678:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787828861.799679:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828861.799681:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1787828861.799683:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828861.799688:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787828861.799689:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369542208 : -126234340009408 : ffff8d30c7015a40) 00000100:00000001:0.0:1787828861.799692:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1787828861.799701:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:0.0:1787828861.799705:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1787828861.799706:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828861.799707:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828861.799708:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787828861.799710:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828861.799713:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298334756 00000100:00000001:0.0:1787828861.799715:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298336000 00000100:00000001:0.0:1787828861.799717:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298334756 00000100:00000001:0.0:1787828861.799718:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298336000 00000100:00000001:0.0:1787828861.799719:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787828861.799721:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828861.799722:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787828861.799724:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828861/real 1787828861] req@ffff8d30c7015a40 x1874670646772480/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828886 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787828861.799729:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787828861.799729:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787828861.799731:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787828861.799732:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787828861.799733:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828861.799734:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828861.799735:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787828861.799738:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c7015a40 x1874670646772480/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828886 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787828861.799743:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828861.799744:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787828861.799745:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828861.799746:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828861.799747:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828861.799748:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787828861.799749:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787828861.799751:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828861.799753:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787828861.799754:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787828861.799756:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c7015a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646772480:192.168.203.143@tcp:8:kworker.0 00000100:00000001:0.0:1787828861.799758:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828861.799759:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828861.799760:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787828861.799761:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787828861.799762:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787828861.799763:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787828861.799763:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787828861.799765:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787828861.799765:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787828861.799766:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787828861.799767:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828861.799768:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828861.799769:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787828861.799770:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787828861.799771:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787828861.799772:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787828861.799773:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787828861.799774:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787828861.799775:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787828861.799776:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787828861.799776:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828861.799778:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828861.799781:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787828861.799782:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828861.799783:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787828861.799784:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828862.694714:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828862.694721:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828862.694843:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828862.694848:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828862.694851:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828863.718718:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828863.718724:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828863.718784:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828863.718787:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828863.718791:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828864.742667:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828864.742672:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828864.742713:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828864.742734:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828864.742739:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828865.766753:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828865.766763:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828865.766899:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828865.766907:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828865.766929:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828866.790697:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828866.790704:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828866.790757:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828866.790761:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828866.790765:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787828866.918753:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828866.918788:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1787828866.918791:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787828866.918796:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787828866.918801:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787828866.918810:0:40701:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 3667 00000100:00080000:3.0:1787828866.918824:0:40701:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787828866.918827:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787828866.918829:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:3.0:1787828866.918834:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1787828866.918837:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1787828866.918841:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:3.0:1787828866.918859:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:3.0:1787828866.918862:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787828866.918864:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:3.0:1787828866.918869:0:40701:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:3.0:1787828866.918872:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787828866.918875:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787828866.918879:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787828866.918883:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828866.918885:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:3.0:1787828866.918889:0:40701:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787828866.918905:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787828866.918907:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828866.918927:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787828866.918932:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787828866.918937:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828866.918952:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787828866.918963:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c70161c0 x1874670646772608/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787828866.918991:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787828866.918996:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828866.919021:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787828866.919041:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828866.919045:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1787828866.919047:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787828866.919049:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787828866.919053:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787828866.919072:0:40701:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3667 00000100:00080000:3.0:1787828866.919076:0:40701:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787828866.919078:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787828866.919080:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:3.0:1787828866.919082:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1787828866.919083:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1787828866.919086:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000020:00000001:3.0:1787828866.919088:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:3.0:1787828866.919090:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787828866.919092:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:3.0:1787828866.919095:0:40701:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:3.0:1787828866.919098:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828866.919100:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:3.0:1787828866.919100:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:0.0:1787828866.919108:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 02000000:00000001:3.0:1787828866.919108:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787828866.919112:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 02000000:00000001:3.0:1787828866.919113:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828866.919117:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1787828866.919118:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787828866.919121:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000008:00100000:3.0:1787828866.919123:0:40701:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 02000000:00000001:0.0:1787828866.919127:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000008:00000001:3.0:1787828866.919128:0:40701:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828866.919133:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787828866.919134:0:40701:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828866.919141:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787828866.919142:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787828866.919145:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828866.919149:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00100000:0.0:1787828866.919150:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c70161c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646772608:192.168.203.143@tcp:250:kworker.0 02000000:00000001:3.0:1787828866.919151:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787828866.919156:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828866.919161:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:3.0:1787828866.919164:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828866.919168:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787828866.919172:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787828866.919174:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c7016580 x1874670646772736/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:0.0:1787828866.919176:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1787828866.919183:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828866.919185:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787828866.919189:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828866.919193:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787828866.919195:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828866.919199:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1787828866.919200:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787828866.919201:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787828866.919204:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:0.0:1787828866.919206:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00080000:3.0:1787828866.919207:0:40701:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3667 00000100:00000001:0.0:1787828866.919210:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369544128 : -126234340007488 : ffff8d30c70161c0) 00000100:00080000:3.0:1787828866.919213:0:40701:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787828866.919216:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:3.0:1787828866.919217:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787828866.919221:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:3.0:1787828866.919223:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1787828866.919224:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1787828866.919226:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000020:00000001:3.0:1787828866.919228:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:3.0:1787828866.919230:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787828866.919231:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:3.0:1787828866.919234:0:40701:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:3.0:1787828866.919236:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787828866.919237:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787828866.919240:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787828866.919242:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828866.919244:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:3.0:1787828866.919245:0:40701:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1787828866.919247:0:40701:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828866.919249:0:40701:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787828866.919255:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787828866.919257:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828866.919258:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787828866.919259:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787828866.919261:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828866.919266:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:0.0:1787828866.919267:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00100000:3.0:1787828866.919275:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c7015a40 x1874670646772864/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787828866.919283:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787828866.919286:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787828866.919286:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828866.919293:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828866.919295:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00080000:3.0:1787828866.919296:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1787828866.919301:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787828866.919301:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1787828866.919306:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1787828866.919307:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787828866.919311:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787828866.919314:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:0.0:1787828866.919316:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787828866.919317:0:40701:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3667 00000100:00000001:0.0:1787828866.919323:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4298341120 00000100:00080000:3.0:1787828866.919324:0:40701:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787828866.919329:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787828866.919331:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:3.0:1787828866.919333:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1787828866.919334:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1787828866.919335:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000020:00000001:3.0:1787828866.919337:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:0.0:1787828866.919340:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 00000100:00000001:3.0:1787828866.919341:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787828866.919343:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:3.0:1787828866.919348:0:40701:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787828866.919351:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4298341120 00000100:00000001:3.0:1787828866.919352:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828866.919357:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 02000000:00000001:3.0:1787828866.919357:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787828866.919360:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787828866.919363:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787828866.919364:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828866.919370:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1787828866.919371:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000008:00100000:3.0:1787828866.919373:0:40701:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00000001:0.0:1787828866.919376:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000008:00000001:3.0:1787828866.919377:0:40701:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828866.919380:0:40701:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787828866.919385:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00100000:0.0:1787828866.919386:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828866/real 1787828866] req@ffff8d30c70161c0 x1874670646772608/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828891 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1787828866.919387:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828866.919393:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787828866.919395:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787828866.919398:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828866.919406:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787828866.919416:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c7014f00 x1874670646772992/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000001:0.0:1787828866.919420:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787828866.919425:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787828866.919426:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787828866.919429:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787828866.919430:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828866.919436:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787828866.919436:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828866.919442:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828866.919446:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828866.919448:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787828866.919454:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c70161c0 x1874670646772608/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828891 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787828866.919462:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828866.919466:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787828866.919469:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828866.919470:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828866.919471:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828866.919473:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787828866.919475:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787828866.919478:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828866.919481:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787828866.919483:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787828866.919497:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c70161c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646772608:192.168.203.143@tcp:250:kworker.0 00000100:00000001:0.0:1787828866.919506:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828866.919507:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828866.919509:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787828866.919511:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787828866.919514:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787828866.919516:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787828866.919518:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787828866.919523:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787828866.919525:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787828866.919527:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787828866.919529:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828866.919531:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828866.919541:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787828866.919543:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828866.919545:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787828866.919547:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787828866.919548:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1787828866.919549:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787828866.919550:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828866.919552:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787828866.919554:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787828866.919556:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828866.919559:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c7016580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646772736:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787828866.919562:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787828866.919564:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787828866.919565:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828866.919567:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1787828866.919569:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828866.919574:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787828866.919575:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369545088 : -126234340006528 : ffff8d30c7016580) 00000100:00000001:0.0:1787828866.919598:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1787828866.919611:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:0.0:1787828866.919641:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1787828866.919644:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828866.919645:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828866.919647:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787828866.919650:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828866.919656:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298339863 00000100:00000001:0.0:1787828866.919659:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298341120 00000100:00000001:0.0:1787828866.919661:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298339863 00000100:00000001:0.0:1787828866.919663:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298341120 00000100:00000001:0.0:1787828866.919665:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787828866.919667:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828866.919669:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787828866.919673:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828866/real 1787828866] req@ffff8d30c7016580 x1874670646772736/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828891 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787828866.919688:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787828866.919689:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787828866.919692:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787828866.919693:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787828866.919694:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828866.919696:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828866.919697:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787828866.919701:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c7016580 x1874670646772736/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828891 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787828866.919708:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828866.919710:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787828866.919712:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828866.919713:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828866.919714:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828866.919716:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787828866.919717:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787828866.919720:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828866.919722:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787828866.919724:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787828866.919727:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c7016580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646772736:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787828866.919730:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1787828866.919731:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787828866.919733:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828866.919735:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787828866.919737:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787828866.919739:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828866.919741:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c7015a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646772864:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787828866.919744:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787828866.919746:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787828866.919748:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828866.919750:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1787828866.919753:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828866.919757:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787828866.919758:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369542208 : -126234340009408 : ffff8d30c7015a40) 00000100:00000001:0.0:1787828866.919796:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1787828866.919808:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:0.0:1787828866.919812:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1787828866.919814:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828866.919816:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828866.919817:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787828866.919819:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828866.919822:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298339863 00000100:00000001:0.0:1787828866.919824:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298341120 00000100:00000001:0.0:1787828866.919827:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298339863 00000100:00000001:0.0:1787828866.919829:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298341120 00000100:00000001:0.0:1787828866.919831:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787828866.919833:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828866.919835:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787828866.919838:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828866/real 1787828866] req@ffff8d30c7015a40 x1874670646772864/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828891 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787828866.919861:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787828866.919862:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787828866.919864:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787828866.919865:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787828866.919866:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828866.919868:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828866.919869:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787828866.919873:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c7015a40 x1874670646772864/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828891 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787828866.919879:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828866.919881:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787828866.919884:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828866.919886:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828866.919887:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828866.919889:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787828866.919890:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787828866.919893:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828866.919895:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787828866.919897:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787828866.919900:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c7015a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646772864:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787828866.919903:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1787828866.919904:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787828866.919905:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828866.919923:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787828866.919925:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787828866.919926:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828866.919929:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c7014f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646772992:192.168.203.143@tcp:8:kworker.0 00000100:00000001:0.0:1787828866.919932:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787828866.919934:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787828866.919936:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828866.919938:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1787828866.919942:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828866.919950:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787828866.919952:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369539328 : -126234340012288 : ffff8d30c7014f00) 00000100:00000001:0.0:1787828866.919957:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1787828866.919972:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:0.0:1787828866.919994:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1787828866.919997:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828866.919999:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828866.920002:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787828866.920006:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828866.920011:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298339876 00000100:00000001:0.0:1787828866.920014:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298341120 00000100:00000001:0.0:1787828866.920018:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298339876 00000100:00000001:0.0:1787828866.920022:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298341120 00000100:00000001:0.0:1787828866.920215:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787828866.920217:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828866.920219:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787828866.920224:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828866/real 1787828866] req@ffff8d30c7014f00 x1874670646772992/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828891 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787828866.920232:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787828866.920233:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787828866.920236:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787828866.920238:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787828866.920239:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828866.920241:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828866.920242:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787828866.920245:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c7014f00 x1874670646772992/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828891 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787828866.920251:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828866.920253:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787828866.920255:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828866.920256:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828866.920257:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828866.920259:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787828866.920260:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787828866.920262:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828866.920264:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787828866.920266:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787828866.920268:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c7014f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646772992:192.168.203.143@tcp:8:kworker.0 00000100:00000001:0.0:1787828866.920271:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828866.920272:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828866.920274:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787828866.920276:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787828866.920278:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787828866.920279:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787828866.920281:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787828866.920283:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787828866.920284:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787828866.920286:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787828866.920287:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828866.920288:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828866.920289:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787828866.920291:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787828866.920292:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787828866.920293:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787828866.920294:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787828866.920296:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787828866.920297:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787828866.920298:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787828866.920299:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828866.920300:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828866.920301:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787828866.920302:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787828866.920303:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787828866.920304:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787828866.920305:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787828866.920306:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787828866.920307:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787828866.920308:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787828866.920309:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828866.920311:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828866.920313:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787828866.920314:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828866.920316:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787828866.920317:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828867.814704:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828867.814712:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828867.814796:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828867.814802:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828867.814807:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828868.838706:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828868.838714:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828868.838774:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828868.838780:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828868.838784:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828869.862667:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828869.862672:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828869.862714:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828869.862718:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828869.862722:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828870.886674:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828870.886682:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828870.886742:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828870.886747:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828870.886751:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828871.910687:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828871.910693:0:815:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1787828871.910695:0:815:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1787828871.910698:0:815:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828871.910701:0:815:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1787828871.910703:0:815:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828871.910705:0:815:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828871.910706:0:815:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828871.910748:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828871.910751:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828871.910754:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787828872.038721:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828872.038732:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1787828872.038735:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787828872.038739:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787828872.038743:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787828872.038750:0:40701:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 3672 00000100:00080000:3.0:1787828872.038762:0:40701:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787828872.038765:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787828872.038767:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:3.0:1787828872.038771:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1787828872.038774:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1787828872.038777:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:3.0:1787828872.038779:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:3.0:1787828872.038782:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787828872.038783:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:3.0:1787828872.038788:0:40701:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:3.0:1787828872.038791:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787828872.038793:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787828872.038797:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787828872.038800:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828872.038802:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:3.0:1787828872.038805:0:40701:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787828872.038816:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787828872.038818:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828872.038821:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787828872.038824:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787828872.038828:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828872.038838:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787828872.038847:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c70143c0 x1874670646773120/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787828872.038858:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787828872.038862:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828872.038879:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787828872.038882:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828872.038885:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1787828872.038887:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787828872.038889:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787828872.038892:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787828872.038895:0:40701:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3672 00000100:00080000:3.0:1787828872.038899:0:40701:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787828872.038902:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787828872.038903:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:3.0:1787828872.038906:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1787828872.038908:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1787828872.038910:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:0.0:1787828872.038911:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:3.0:1787828872.038913:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:0.0:1787828872.038917:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787828872.038918:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787828872.038920:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787828872.038922:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:0.0:1787828872.038924:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787828872.038927:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787828872.038932:0:40701:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 02000000:00000001:0.0:1787828872.038933:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787828872.038939:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787828872.038939:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828872.038945:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787828872.038946:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00100000:0.0:1787828872.038951:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c70143c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646773120:192.168.203.143@tcp:250:kworker.0 02000000:00000001:3.0:1787828872.038955:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787828872.038956:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1787828872.038961:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828872.038962:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787828872.038965:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828872.038967:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1787828872.038971:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828872.038981:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:3.0:1787828872.038985:0:40701:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00000001:0.0:1787828872.038986:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000008:00000001:3.0:1787828872.038998:0:40701:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828872.039001:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369536448 : -126234340015168 : ffff8d30c70143c0) 00000100:00000001:3.0:1787828872.039002:0:40701:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828872.039010:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 02000000:00000001:3.0:1787828872.039011:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787828872.039014:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828872.039017:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787828872.039019:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787828872.039021:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828872.039028:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787828872.039035:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c7015680 x1874670646773248/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787828872.039047:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:3.0:1787828872.039056:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787828872.039060:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828872.039063:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787828872.039064:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787828872.039069:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1787828872.039071:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828872.039075:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787828872.039076:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1787828872.039082:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1787828872.039082:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787828872.039086:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787828872.039089:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:0.0:1787828872.039091:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787828872.039092:0:40701:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3672 00000100:00000001:0.0:1787828872.039100:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4298346240 00000100:00080000:3.0:1787828872.039100:0:40701:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787828872.039105:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787828872.039107:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:3.0:1787828872.039109:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1787828872.039110:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1787828872.039111:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:0.0:1787828872.039112:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 00000020:00000001:3.0:1787828872.039113:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:3.0:1787828872.039117:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787828872.039119:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:0.0:1787828872.039122:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4298346240 00000100:00080000:3.0:1787828872.039122:0:40701:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:3.0:1787828872.039128:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828872.039129:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 02000000:00000001:3.0:1787828872.039130:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:0.0:1787828872.039133:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787828872.039135:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787828872.039136:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787828872.039141:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828872.039142:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00000001:3.0:1787828872.039145:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:3.0:1787828872.039148:0:40701:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787828872.039151:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828872/real 1787828872] req@ffff8d30c70143c0 x1874670646773120/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828897 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00000001:3.0:1787828872.039152:0:40701:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828872.039156:0:40701:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787828872.039161:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787828872.039164:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:0.0:1787828872.039167:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000001:3.0:1787828872.039167:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000400:00000001:0.0:1787828872.039171:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 02000000:00000001:3.0:1787828872.039171:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787828872.039175:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828872.039176:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787828872.039180:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787828872.039180:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828872.039184:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828872.039186:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00100000:3.0:1787828872.039188:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c70152c0 x1874670646773376/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787828872.039195:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787828872.039197:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787828872.039198:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828872.039204:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828872.039208:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c70143c0 x1874670646773120/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828897 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828872.039208:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828872.039213:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1787828872.039217:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787828872.039219:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787828872.039224:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787828872.039240:0:40701:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3672 00000100:00080000:3.0:1787828872.039245:0:40701:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787828872.039248:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787828872.039248:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787828872.039251:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:0.0:1787828872.039253:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000020:00000001:3.0:1787828872.039254:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1787828872.039257:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:0.0:1787828872.039258:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:3.0:1787828872.039259:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:0.0:1787828872.039261:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828872.039264:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:3.0:1787828872.039265:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:0.0:1787828872.039267:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787828872.039268:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787828872.039270:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787828872.039272:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:0.0:1787828872.039274:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828872.039278:0:40701:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00080000:0.0:1787828872.039282:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787828872.039283:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828872.039285:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 02000000:00000001:3.0:1787828872.039286:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787828872.039290:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00100000:0.0:1787828872.039291:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c70143c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646773120:192.168.203.143@tcp:250:kworker.0 02000000:00000001:3.0:1787828872.039295:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828872.039298:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787828872.039299:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:3.0:1787828872.039302:0:40701:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00000001:0.0:1787828872.039303:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828872.039306:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 00000008:00000001:3.0:1787828872.039307:0:40701:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828872.039310:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 00000100:00000001:3.0:1787828872.039310:0:40701:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828872.039314:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 02000000:00000001:3.0:1787828872.039315:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000020:00000001:0.0:1787828872.039317:0:2390:0:(genops.c:1217:class_import_put()) Process entered 02000000:00000001:3.0:1787828872.039318:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787828872.039321:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 00000100:00000001:3.0:1787828872.039321:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787828872.039325:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787828872.039326:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828872.039327:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 00000100:00000001:3.0:1787828872.039329:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828872.039330:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787828872.039335:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787828872.039337:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787828872.039338:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c7014b40 x1874670646773504/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787828872.039339:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787828872.039348:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787828872.039349:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787828872.039350:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828872.039352:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828872.039354:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828872.039355:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787828872.039359:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787828872.039360:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1787828872.039361:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787828872.039363:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828872.039364:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787828872.039366:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787828872.039367:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828872.039371:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c7015680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646773248:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787828872.039374:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787828872.039376:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787828872.039377:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828872.039379:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1787828872.039381:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828872.039385:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787828872.039386:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369541248 : -126234340010368 : ffff8d30c7015680) 00000100:00000001:0.0:1787828872.039390:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1787828872.039400:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:0.0:1787828872.039403:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1787828872.039405:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828872.039406:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828872.039407:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787828872.039409:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828872.039412:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298344982 00000100:00000001:0.0:1787828872.039414:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298346240 00000100:00000001:0.0:1787828872.039416:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298344982 00000100:00000001:0.0:1787828872.039418:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298346240 00000100:00000001:0.0:1787828872.039419:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787828872.039420:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828872.039421:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787828872.039424:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828872/real 1787828872] req@ffff8d30c7015680 x1874670646773248/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828897 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787828872.039429:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787828872.039430:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787828872.039432:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787828872.039433:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787828872.039434:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828872.039435:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828872.039436:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787828872.039438:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c7015680 x1874670646773248/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828897 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787828872.039447:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828872.039449:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787828872.039450:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828872.039451:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828872.039452:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828872.039453:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787828872.039454:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787828872.039456:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828872.039458:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787828872.039459:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787828872.039461:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c7015680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646773248:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787828872.039463:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1787828872.039464:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787828872.039465:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828872.039466:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787828872.039467:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787828872.039468:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828872.039470:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c70152c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646773376:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787828872.039472:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787828872.039473:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787828872.039475:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828872.039476:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1787828872.039478:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828872.039481:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787828872.039482:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369540288 : -126234340011328 : ffff8d30c70152c0) 00000100:00000001:0.0:1787828872.039484:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1787828872.039490:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:0.0:1787828872.039492:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1787828872.039494:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828872.039494:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828872.039496:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787828872.039498:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828872.039500:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298344982 00000100:00000001:0.0:1787828872.039502:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298346240 00000100:00000001:0.0:1787828872.039515:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298344982 00000100:00000001:0.0:1787828872.039516:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298346240 00000100:00000001:0.0:1787828872.039517:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787828872.039519:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828872.039520:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787828872.039522:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828872/real 1787828872] req@ffff8d30c70152c0 x1874670646773376/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828897 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787828872.039526:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787828872.039527:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787828872.039528:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787828872.039529:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787828872.039530:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828872.039531:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828872.039532:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787828872.039534:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c70152c0 x1874670646773376/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828897 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787828872.039538:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828872.039540:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787828872.039541:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828872.039542:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828872.039543:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828872.039544:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787828872.039545:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787828872.039547:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828872.039548:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787828872.039549:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787828872.039552:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c70152c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646773376:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787828872.039554:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1787828872.039555:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787828872.039556:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828872.039557:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787828872.039558:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787828872.039559:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828872.039561:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c7014b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646773504:192.168.203.143@tcp:8:kworker.0 00000100:00000001:0.0:1787828872.039563:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787828872.039564:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787828872.039565:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828872.039566:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1787828872.039567:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828872.039570:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787828872.039571:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369538368 : -126234340013248 : ffff8d30c7014b40) 00000100:00000001:0.0:1787828872.039573:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1787828872.039578:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:0.0:1787828872.039580:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1787828872.039581:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828872.039582:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828872.039583:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787828872.039585:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828872.039587:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298344995 00000100:00000001:0.0:1787828872.039589:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298346240 00000100:00000001:0.0:1787828872.039590:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298344995 00000100:00000001:0.0:1787828872.039592:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298346240 00000100:00000001:0.0:1787828872.039593:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787828872.039594:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828872.039595:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787828872.039597:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828872/real 1787828872] req@ffff8d30c7014b40 x1874670646773504/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828897 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787828872.039629:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787828872.039630:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787828872.039632:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787828872.039633:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787828872.039634:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828872.039635:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828872.039636:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787828872.039639:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c7014b40 x1874670646773504/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828897 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787828872.039643:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828872.039645:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787828872.039646:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828872.039647:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828872.039648:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828872.039649:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787828872.039650:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787828872.039652:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828872.039653:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787828872.039655:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787828872.039656:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c7014b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646773504:192.168.203.143@tcp:8:kworker.0 00000100:00000001:0.0:1787828872.039659:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828872.039660:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828872.039661:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787828872.039662:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787828872.039663:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787828872.039664:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787828872.039665:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787828872.039667:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787828872.039667:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787828872.039669:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787828872.039670:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828872.039671:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828872.039672:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787828872.039673:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787828872.039674:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787828872.039675:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787828872.039676:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787828872.039677:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787828872.039677:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787828872.039678:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787828872.039679:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828872.039680:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828872.039681:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787828872.039681:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787828872.039682:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787828872.039683:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787828872.039683:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787828872.039684:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787828872.039685:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787828872.039686:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787828872.039687:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828872.039688:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828872.039690:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787828872.039690:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828872.039692:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787828872.039693:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828872.934687:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828872.934691:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828872.934754:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828872.934758:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828872.934761:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828873.958697:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828873.958702:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828873.958763:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828873.958767:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828873.958771:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828874.982723:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828874.982729:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828874.982784:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828874.982789:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828874.982793:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828876.006718:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828876.006724:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828876.006794:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828876.006798:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828876.006801:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828877.030769:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828877.030775:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828877.030892:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828877.030896:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828877.030899:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787828877.158683:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828877.158690:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1787828877.158692:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787828877.158696:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787828877.158699:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787828877.158705:0:40701:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 3677 00000100:00080000:3.0:1787828877.158715:0:40701:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787828877.158718:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787828877.158719:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:3.0:1787828877.158722:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1787828877.158725:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1787828877.158727:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:3.0:1787828877.158729:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:3.0:1787828877.158731:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787828877.158731:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:3.0:1787828877.158735:0:40701:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:3.0:1787828877.158736:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787828877.158738:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787828877.158741:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787828877.158743:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828877.158744:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:3.0:1787828877.158746:0:40701:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787828877.158756:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787828877.158757:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828877.158760:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787828877.158762:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787828877.158766:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828877.158775:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787828877.158784:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c7016d00 x1874670646773632/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787828877.158795:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787828877.158799:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828877.158820:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787828877.158824:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828877.158831:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1787828877.158833:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787828877.158835:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787828877.158837:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787828877.158841:0:40701:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3677 00000100:00080000:3.0:1787828877.158846:0:40701:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787828877.158848:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787828877.158850:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:3.0:1787828877.158853:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1787828877.158854:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1787828877.158856:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000020:00000001:3.0:1787828877.158859:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:0.0:1787828877.158862:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1787828877.158862:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787828877.158867:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:0.0:1787828877.158868:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787828877.158872:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00080000:3.0:1787828877.158873:0:40701:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787828877.158875:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787828877.158878:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828877.158879:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828877.158882:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787828877.158884:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787828877.158886:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787828877.158889:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787828877.158891:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00100000:0.0:1787828877.158893:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c7016d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646773632:192.168.203.143@tcp:250:kworker.0 02000000:00000001:3.0:1787828877.158897:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828877.158899:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:3.0:1787828877.158902:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 02000000:00000001:0.0:1787828877.158903:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787828877.158907:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000008:00100000:3.0:1787828877.158908:0:40701:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 02000000:00000001:0.0:1787828877.158909:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000008:00000001:3.0:1787828877.158914:0:40701:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828877.158915:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828877.158919:0:40701:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787828877.158926:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787828877.158928:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828877.158930:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787828877.158931:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:0.0:1787828877.158933:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369547008 : -126234340004608 : ffff8d30c7016d00) 02000000:00000001:3.0:1787828877.158935:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:0.0:1787828877.158937:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 02000000:00000001:3.0:1787828877.158939:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828877.158947:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787828877.158968:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c70170c0 x1874670646773760/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787828877.158975:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787828877.158977:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828877.158979:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787828877.158982:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:0.0:1787828877.158986:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00080000:3.0:1787828877.158987:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1787828877.158991:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787828877.158993:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787828877.158995:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:0.0:1787828877.159000:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00080000:3.0:1787828877.159000:0:40701:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3677 00000100:00000001:0.0:1787828877.159006:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00080000:3.0:1787828877.159008:0:40701:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787828877.159009:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828877.159014:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1787828877.159014:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787828877.159018:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:0.0:1787828877.159019:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828877.159022:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:0.0:1787828877.159024:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4298351360 00000020:00000001:3.0:1787828877.159026:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1787828877.159031:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:0.0:1787828877.159033:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 00000020:00000001:3.0:1787828877.159034:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:3.0:1787828877.159039:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787828877.159041:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4298351360 00000100:00000001:3.0:1787828877.159042:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:0.0:1787828877.159047:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 00000100:00080000:3.0:1787828877.159051:0:40701:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787828877.159052:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787828877.159057:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828877.159057:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828877.159062:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 02000000:00000001:3.0:1787828877.159062:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787828877.159066:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00100000:0.0:1787828877.159070:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828877/real 1787828877] req@ffff8d30c7016d00 x1874670646773632/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828902 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1787828877.159071:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828877.159075:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:3.0:1787828877.159079:0:40701:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000400:00000001:0.0:1787828877.159084:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787828877.159086:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787828877.159088:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000008:00000001:3.0:1787828877.159090:0:40701:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828877.159091:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787828877.159093:0:40701:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828877.159094:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828877.159097:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828877.159099:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:3.0:1787828877.159099:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787828877.159102:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828877.159106:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c7016d00 x1874670646773632/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828902 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787828877.159106:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787828877.159110:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787828877.159113:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828877.159119:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787828877.159119:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828877.159126:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00100000:3.0:1787828877.159128:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c7017840 x1874670646773888/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787828877.159129:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828877.159134:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828877.159136:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828877.159140:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787828877.159144:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787828877.159145:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787828877.159149:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787828877.159150:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00000001:3.0:1787828877.159153:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787828877.159156:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00080000:3.0:1787828877.159158:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1787828877.159159:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00080000:3.0:1787828877.159162:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:0.0:1787828877.159166:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c7016d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646773632:192.168.203.143@tcp:250:kworker.0 00000100:00000001:3.0:1787828877.159166:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787828877.159170:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787828877.159172:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787828877.159173:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:0.0:1787828877.159175:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828877.159179:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 00000100:00080000:3.0:1787828877.159179:0:40701:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3677 02000000:00000001:0.0:1787828877.159183:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 00000100:00080000:3.0:1787828877.159185:0:40701:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 02000000:00000001:0.0:1787828877.159186:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00000001:3.0:1787828877.159187:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000020:00000001:0.0:1787828877.159189:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000100:00000001:3.0:1787828877.159190:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787828877.159192:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 00000020:00000001:3.0:1787828877.159194:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1787828877.159196:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 02000000:00000001:0.0:1787828877.159197:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 00000020:00000001:3.0:1787828877.159198:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 02000000:00000001:0.0:1787828877.159199:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000020:00000001:3.0:1787828877.159203:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:0.0:1787828877.159204:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787828877.159205:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787828877.159206:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787828877.159207:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:0.0:1787828877.159209:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:3.0:1787828877.159213:0:40701:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:3.0:1787828877.159215:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787828877.159216:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:0.0:1787828877.159217:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 02000000:00000001:3.0:1787828877.159218:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787828877.159220:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787828877.159221:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828877.159223:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1787828877.159224:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1787828877.159226:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000008:00100000:3.0:1787828877.159228:0:40701:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00000001:0.0:1787828877.159229:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000008:00000001:3.0:1787828877.159231:0:40701:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828877.159232:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787828877.159235:0:40701:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828877.159236:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828877.159241:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787828877.159243:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787828877.159243:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787828877.159246:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828877.159247:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828877.159250:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787828877.159252:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00100000:0.0:1787828877.159253:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c70170c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646773760:192.168.203.143@tcp:38:kworker.0 02000000:00000001:3.0:1787828877.159255:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828877.159257:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787828877.159260:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787828877.159261:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828877.159261:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828877.159266:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1787828877.159269:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787828877.159270:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c7014000 x1874670646774016/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787828877.159276:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787828877.159278:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787828877.159279:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369547968 : -126234340003648 : ffff8d30c70170c0) 00000100:00000001:3.0:1787828877.159281:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828877.159285:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:3.0:1787828877.159286:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:0.0:1787828877.159298:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:0.0:1787828877.159302:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1787828877.159305:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828877.159306:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828877.159308:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787828877.159311:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828877.159314:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298350102 00000100:00000001:0.0:1787828877.159317:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298351360 00000100:00000001:0.0:1787828877.159319:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298350102 00000100:00000001:0.0:1787828877.159321:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298351360 00000100:00000001:0.0:1787828877.159323:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787828877.159325:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828877.159327:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787828877.159332:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828877/real 1787828877] req@ffff8d30c70170c0 x1874670646773760/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828902 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787828877.159340:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787828877.159342:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787828877.159345:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787828877.159346:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787828877.159347:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828877.159349:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828877.159350:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787828877.159354:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c70170c0 x1874670646773760/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828902 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787828877.159361:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828877.159363:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787828877.159365:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828877.159367:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828877.159368:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828877.159370:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787828877.159372:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787828877.159375:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828877.159379:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787828877.159381:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787828877.159384:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c70170c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646773760:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787828877.159388:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1787828877.159390:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787828877.159392:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828877.159395:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787828877.159398:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787828877.159400:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828877.159404:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c7017840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646773888:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787828877.159407:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787828877.159409:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787828877.159411:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828877.159413:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1787828877.159415:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828877.159421:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787828877.159423:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369549888 : -126234340001728 : ffff8d30c7017840) 00000100:00000001:0.0:1787828877.159427:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1787828877.159441:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:0.0:1787828877.159447:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1787828877.159450:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828877.159452:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828877.159454:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787828877.159457:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828877.159460:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298350102 00000100:00000001:0.0:1787828877.159463:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298351360 00000100:00000001:0.0:1787828877.159465:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298350102 00000100:00000001:0.0:1787828877.159468:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298351360 00000100:00000001:0.0:1787828877.159478:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787828877.159481:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828877.159482:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787828877.159486:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828877/real 1787828877] req@ffff8d30c7017840 x1874670646773888/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828902 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787828877.159492:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787828877.159493:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787828877.159495:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787828877.159496:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787828877.159497:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828877.159498:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828877.159499:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787828877.159502:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c7017840 x1874670646773888/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828902 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787828877.159506:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828877.159508:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787828877.159509:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828877.159510:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828877.159511:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828877.159512:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787828877.159513:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787828877.159515:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828877.159516:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787828877.159518:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787828877.159520:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c7017840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646773888:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787828877.159525:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828877.159527:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828877.159529:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787828877.159530:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787828877.159531:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787828877.159532:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787828877.159534:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787828877.159536:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787828877.159537:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787828877.159538:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787828877.159540:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828877.159542:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828877.159543:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787828877.159544:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787828877.159546:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787828877.159548:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787828877.159549:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787828877.159551:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787828877.159551:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787828877.159553:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787828877.159554:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828877.159555:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828877.159557:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787828877.159558:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828877.159559:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787828877.159560:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787828877.159561:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1787828877.159562:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787828877.159563:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828877.159564:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787828877.159566:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787828877.159567:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828877.159569:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c7014000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646774016:192.168.203.143@tcp:8:kworker.0 00000100:00000001:0.0:1787828877.159571:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787828877.159572:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787828877.159574:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828877.159575:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1787828877.159577:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828877.159581:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787828877.159583:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369535488 : -126234340016128 : ffff8d30c7014000) 00000100:00000001:0.0:1787828877.159586:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1787828877.159598:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:0.0:1787828877.159647:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1787828877.159649:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828877.159650:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828877.159652:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787828877.159654:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828877.159656:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298350116 00000100:00000001:0.0:1787828877.159659:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298351360 00000100:00000001:0.0:1787828877.159660:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298350116 00000100:00000001:0.0:1787828877.159662:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298351360 00000100:00000001:0.0:1787828877.159663:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787828877.159665:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828877.159666:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787828877.159669:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828877/real 1787828877] req@ffff8d30c7014000 x1874670646774016/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828902 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787828877.159674:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787828877.159675:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787828877.159676:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787828877.159677:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787828877.159678:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828877.159679:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828877.159680:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787828877.159683:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c7014000 x1874670646774016/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828902 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787828877.159687:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828877.159689:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787828877.159690:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828877.159691:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828877.159692:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828877.159693:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787828877.159694:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787828877.159696:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828877.159697:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787828877.159698:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787828877.159700:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c7014000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646774016:192.168.203.143@tcp:8:kworker.0 00000100:00000001:0.0:1787828877.159702:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828877.159704:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828877.159705:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787828877.159706:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787828877.159707:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787828877.159708:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787828877.159709:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787828877.159710:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787828877.159711:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787828877.159713:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787828877.159713:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828877.159715:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828877.159717:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787828877.159718:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828877.159719:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787828877.159720:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828878.054691:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828878.054697:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828878.054751:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828878.054755:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828878.054758:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828879.078715:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828879.078721:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828879.078805:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828879.078809:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828879.078813:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828880.102696:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828880.102703:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828880.102764:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828880.102769:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828880.102773:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828881.126703:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828881.126715:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828881.126774:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828881.126778:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828881.126781:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828882.150688:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828882.150695:0:815:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1787828882.150696:0:815:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1787828882.150699:0:815:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828882.150703:0:815:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1787828882.150705:0:815:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828882.150707:0:815:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828882.150709:0:815:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828882.150770:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828882.150773:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828882.150792:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787828882.278737:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828882.278746:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1787828882.278748:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787828882.278751:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787828882.278755:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787828882.278762:0:40701:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 3682 00000100:00080000:0.0:1787828882.278770:0:40701:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787828882.278772:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787828882.278774:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787828882.278778:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787828882.278781:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787828882.278784:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:0.0:1787828882.278787:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:0.0:1787828882.278789:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787828882.278790:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:0.0:1787828882.278795:0:40701:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787828882.278798:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828882.278801:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787828882.278805:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787828882.278809:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828882.278811:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1787828882.278814:0:40701:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828882.278826:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787828882.278828:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828882.278832:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787828882.278834:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787828882.278838:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828882.278849:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828882.278858:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d270c0 x1874670646774144/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787828882.278869:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787828882.278873:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828882.278891:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787828882.278893:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828882.278895:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1787828882.278896:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787828882.278898:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787828882.278905:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787828882.278908:0:40701:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3682 00000100:00080000:0.0:1787828882.278911:0:40701:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787828882.278913:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787828882.278914:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787828882.278928:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787828882.278930:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787828882.278932:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000020:00000001:0.0:1787828882.278934:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:0.0:1787828882.278936:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828882.278936:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787828882.278939:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:2.0:1787828882.278942:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00080000:0.0:1787828882.278944:0:40701:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787828882.278945:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1787828882.278947:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828882.278949:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 02000000:00000001:0.0:1787828882.278950:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:2.0:1787828882.278952:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828882.278956:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828882.278957:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787828882.278961:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828882.278961:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828882.278966:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828882.278967:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1787828882.278970:0:40701:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1787828882.278973:0:40701:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828882.278976:0:40701:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787828882.278979:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d270c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646774144:192.168.203.143@tcp:250:kworker.0 02000000:00000001:0.0:1787828882.278981:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787828882.278984:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828882.278986:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:0.0:1787828882.278987:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787828882.278989:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787828882.278990:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787828882.278991:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828882.278993:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828882.278996:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:0.0:1787828882.278997:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828882.279001:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828882.279005:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d24000 x1874670646774272/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787828882.279012:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787828882.279014:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828882.279015:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787828882.279018:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828882.279018:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601348800 : -126234108202816 : ffff8d30d4d270c0) 00000100:00080000:0.0:1787828882.279023:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:2.0:1787828882.279024:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00080000:0.0:1787828882.279028:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1787828882.279030:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787828882.279031:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787828882.279034:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787828882.279035:0:40701:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3682 00000100:00080000:0.0:1787828882.279039:0:40701:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787828882.279040:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787828882.279041:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787828882.279043:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787828882.279044:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787828882.279052:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000020:00000001:0.0:1787828882.279054:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:0.0:1787828882.279056:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787828882.279057:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:0.0:1787828882.279059:0:40701:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000800:00000100:2.0:1787828882.279064:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:0.0:1787828882.279065:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828882.279068:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787828882.279070:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787828882.279072:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828882.279083:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1787828882.279084:0:40701:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1787828882.279086:0:40701:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828882.279087:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1787828882.279088:0:40701:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828882.279092:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 02000000:00000001:0.0:1787828882.279093:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787828882.279096:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828882.279096:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828882.279102:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:2.0:1787828882.279102:0:2390:0:(events.c:68:request_out_callback()) Process leaving 02000000:00000001:0.0:1787828882.279106:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787828882.279107:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828882.279108:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828882.279111:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828882.279114:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4298356480 00000100:00100000:0.0:1787828882.279117:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d24f00 x1874670646774400/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787828882.279124:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787828882.279124:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1787828882.279128:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828882.279130:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828882.279132:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4298356480 00000100:00080000:0.0:1787828882.279133:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828882.279138:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1787828882.279139:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1787828882.279140:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787828882.279143:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787828882.279143:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787828882.279148:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828882.279149:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:2.0:1787828882.279151:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00080000:0.0:1787828882.279154:0:40701:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3682 00000100:00080000:0.0:1787828882.279158:0:40701:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1787828882.279159:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828882/real 1787828882] req@ffff8d30d4d270c0 x1874670646774144/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828907 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787828882.279160:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787828882.279162:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787828882.279168:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787828882.279171:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787828882.279174:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000400:00000001:2.0:1787828882.279175:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000020:00000001:0.0:1787828882.279179:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000400:00000001:2.0:1787828882.279179:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1787828882.279182:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787828882.279183:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828882.279185:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787828882.279186:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:2.0:1787828882.279187:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:0.0:1787828882.279191:0:40701:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787828882.279192:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828882.279195:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828882.279196:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828882.279199:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00100000:2.0:1787828882.279201:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d270c0 x1874670646774144/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828907 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:0.0:1787828882.279202:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787828882.279209:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828882.279213:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1787828882.279216:0:40701:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00000001:2.0:1787828882.279217:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000008:00000001:0.0:1787828882.279220:0:40701:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828882.279221:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787828882.279223:0:40701:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828882.279226:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:0.0:1787828882.279229:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:2.0:1787828882.279229:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 02000000:00000001:0.0:1787828882.279232:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828882.279233:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828882.279238:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:2.0:1787828882.279239:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 02000000:00000001:0.0:1787828882.279241:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:2.0:1787828882.279242:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 02000000:00000001:0.0:1787828882.279245:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787828882.279246:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00000001:0.0:1787828882.279249:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787828882.279250:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00100000:0.0:1787828882.279257:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d25a40 x1874670646774528/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787828882.279265:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1787828882.279272:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d270c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646774144:192.168.203.143@tcp:250:kworker.0 00000100:00000001:0.0:1787828882.279274:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787828882.279277:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828882.279278:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828882.279279:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828882.279282:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828882.279284:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787828882.279286:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1787828882.279289:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787828882.279290:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:2.0:1787828882.279292:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:2.0:1787828882.279295:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787828882.279297:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1787828882.279300:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787828882.279301:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828882.279303:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828882.279309:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1787828882.279310:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828882.279311:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787828882.279312:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1787828882.279319:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1787828882.279320:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787828882.279321:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828882.279323:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787828882.279324:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828882.279326:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787828882.279328:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d24000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646774272:192.168.203.143@tcp:38:kworker.0 00000100:00000001:2.0:1787828882.279330:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787828882.279331:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1787828882.279333:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828882.279334:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1787828882.279336:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828882.279340:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787828882.279341:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601336320 : -126234108215296 : ffff8d30d4d24000) 00000100:00000001:2.0:1787828882.279344:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1787828882.279353:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:2.0:1787828882.279356:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1787828882.279357:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828882.279358:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828882.279360:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1787828882.279361:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828882.279364:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298355222 00000100:00000001:2.0:1787828882.279366:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298356480 00000100:00000001:2.0:1787828882.279368:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298355222 00000100:00000001:2.0:1787828882.279370:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298356480 00000100:00000001:2.0:1787828882.279372:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787828882.279373:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828882.279374:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1787828882.279377:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828882/real 1787828882] req@ffff8d30d4d24000 x1874670646774272/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828907 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1787828882.279383:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787828882.279383:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1787828882.279385:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1787828882.279386:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787828882.279387:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828882.279388:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828882.279389:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787828882.279392:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d24000 x1874670646774272/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828907 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787828882.279396:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828882.279398:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787828882.279399:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828882.279400:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828882.279401:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828882.279403:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787828882.279404:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787828882.279405:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828882.279407:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:2.0:1787828882.279409:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1787828882.279411:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d24000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646774272:192.168.203.143@tcp:38:kworker.0 00000100:00000001:2.0:1787828882.279413:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1787828882.279414:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787828882.279415:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828882.279416:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787828882.279417:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828882.279419:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787828882.279421:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d24f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646774400:192.168.203.143@tcp:38:kworker.0 00000100:00000001:2.0:1787828882.279422:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787828882.279423:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1787828882.279424:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828882.279431:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1787828882.279433:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828882.279436:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787828882.279437:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601340160 : -126234108211456 : ffff8d30d4d24f00) 00000100:00000001:2.0:1787828882.279439:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1787828882.279446:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:2.0:1787828882.279449:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1787828882.279450:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828882.279451:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828882.279452:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1787828882.279454:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828882.279456:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298355222 00000100:00000001:2.0:1787828882.279457:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298356480 00000100:00000001:2.0:1787828882.279459:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298355222 00000100:00000001:2.0:1787828882.279460:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298356480 00000100:00000001:2.0:1787828882.279462:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787828882.279463:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828882.279464:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1787828882.279467:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828882/real 1787828882] req@ffff8d30d4d24f00 x1874670646774400/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828907 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1787828882.279471:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787828882.279472:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1787828882.279474:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1787828882.279475:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787828882.279476:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828882.279477:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828882.279478:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787828882.279481:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d24f00 x1874670646774400/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828907 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787828882.279485:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828882.279487:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787828882.279488:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828882.279489:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828882.279490:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828882.279491:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787828882.279492:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787828882.279493:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828882.279495:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:2.0:1787828882.279496:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1787828882.279498:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d24f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646774400:192.168.203.143@tcp:38:kworker.0 00000100:00000001:2.0:1787828882.279500:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1787828882.279501:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787828882.279502:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828882.279503:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787828882.279504:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828882.279506:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787828882.279508:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d25a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646774528:192.168.203.143@tcp:8:kworker.0 00000100:00000001:2.0:1787828882.279510:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787828882.279511:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1787828882.279512:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828882.279513:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1787828882.279514:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828882.279517:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787828882.279518:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601343040 : -126234108208576 : ffff8d30d4d25a40) 00000100:00000001:2.0:1787828882.279520:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1787828882.279526:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:2.0:1787828882.279528:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1787828882.279530:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828882.279531:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828882.279532:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1787828882.279533:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828882.279535:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298355235 00000100:00000001:2.0:1787828882.279536:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298356480 00000100:00000001:2.0:1787828882.279538:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298355235 00000100:00000001:2.0:1787828882.279540:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298356480 00000100:00000001:2.0:1787828882.279541:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787828882.279542:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828882.279543:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1787828882.279546:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828882/real 1787828882] req@ffff8d30d4d25a40 x1874670646774528/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828907 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1787828882.279550:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787828882.279551:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1787828882.279552:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1787828882.279553:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787828882.279554:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828882.279555:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828882.279556:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787828882.279558:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d25a40 x1874670646774528/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828907 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787828882.279562:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828882.279563:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787828882.279565:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828882.279566:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828882.279567:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828882.279568:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787828882.279569:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787828882.279570:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828882.279573:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:2.0:1787828882.279574:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1787828882.279576:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d25a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646774528:192.168.203.143@tcp:8:kworker.0 00000100:00000001:2.0:1787828882.279578:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828882.279580:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828882.279580:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787828882.279581:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1787828882.279583:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787828882.279583:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:2.0:1787828882.279584:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:2.0:1787828882.279585:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787828882.279586:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1787828882.279588:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787828882.279588:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828882.279589:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828882.279590:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787828882.279591:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1787828882.279592:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787828882.279593:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:2.0:1787828882.279594:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:2.0:1787828882.279595:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787828882.279596:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1787828882.279598:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787828882.279624:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828882.279625:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828882.279626:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787828882.279627:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1787828882.279628:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787828882.279629:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:2.0:1787828882.279630:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:2.0:1787828882.279631:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787828882.279632:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1787828882.279633:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787828882.279634:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828882.279635:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828882.279637:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1787828882.279638:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828882.279640:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787828882.279641:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828883.174719:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828883.174724:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828883.174799:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828883.174806:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828883.174810:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828884.198754:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828884.198766:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828884.198844:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828884.198852:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828884.198858:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828885.222701:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828885.222711:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828885.222825:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828885.222830:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828885.222834:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828886.246768:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828886.246779:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828886.246968:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828886.246973:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828886.246977:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828887.270686:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828887.270691:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828887.270770:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828887.270774:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828887.270777:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787828887.398700:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828887.398712:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1787828887.398715:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787828887.398720:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787828887.398725:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787828887.398733:0:40701:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 3687 00000100:00080000:0.0:1787828887.398747:0:40701:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787828887.398753:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787828887.398756:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787828887.398762:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787828887.398766:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787828887.398770:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:0.0:1787828887.398773:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:0.0:1787828887.398775:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787828887.398778:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:0.0:1787828887.398783:0:40701:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787828887.398787:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828887.398790:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787828887.398794:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787828887.398797:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828887.398799:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1787828887.398802:0:40701:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828887.398813:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787828887.398815:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828887.398818:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787828887.398821:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787828887.398825:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828887.398836:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828887.398845:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d25a40 x1874670646774656/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787828887.398854:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787828887.398857:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828887.398872:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787828887.398874:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828887.398877:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1787828887.398879:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787828887.398894:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787828887.398897:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787828887.398900:0:40701:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3687 00000100:00080000:0.0:1787828887.398921:0:40701:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787828887.398923:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787828887.398924:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787828887.398926:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787828887.398927:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787828887.398929:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000020:00000001:0.0:1787828887.398930:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:0.0:1787828887.398932:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787828887.398934:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:0.0:1787828887.398936:0:40701:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787828887.398938:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828887.398940:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787828887.398956:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787828887.398958:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828887.398960:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1787828887.398964:0:40701:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1787828887.398966:0:40701:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828887.398968:0:40701:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828887.398989:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787828887.398991:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828887.398992:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787828887.398993:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787828887.398997:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787828887.399000:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828887.399002:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787828887.399007:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828887.399008:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1787828887.399015:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00100000:0.0:1787828887.399017:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d24f00 x1874670646774784/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787828887.399020:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828887.399030:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:0.0:1787828887.399031:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787828887.399035:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828887.399036:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787828887.399039:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787828887.399046:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 02000000:00000001:2.0:1787828887.399046:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787828887.399055:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1787828887.399058:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787828887.399060:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1787828887.399062:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d25a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646774656:192.168.203.143@tcp:250:kworker.0 00000100:00000001:0.0:1787828887.399063:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787828887.399069:0:40701:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3687 00000100:00080000:0.0:1787828887.399078:0:40701:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787828887.399078:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:0.0:1787828887.399082:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787828887.399083:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787828887.399085:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787828887.399086:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787828887.399088:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000020:00000001:0.0:1787828887.399089:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:0.0:1787828887.399091:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787828887.399092:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:0.0:1787828887.399096:0:40701:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787828887.399098:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828887.399099:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787828887.399104:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787828887.399106:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828887.399107:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787828887.399115:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828887.399116:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:0.0:1787828887.399117:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1787828887.399121:0:40701:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1787828887.399123:0:40701:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828887.399124:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828887.399125:0:40701:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828887.399132:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787828887.399134:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828887.399135:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787828887.399136:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787828887.399155:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828887.399159:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787828887.399160:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828887.399162:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601343040 : -126234108208576 : ffff8d30d4d25a40) 00000100:00100000:0.0:1787828887.399167:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d24000 x1874670646774912/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787828887.399168:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:0.0:1787828887.399176:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787828887.399178:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828887.399181:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787828887.399183:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828887.399186:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1787828887.399187:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787828887.399188:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787828887.399189:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787828887.399191:0:40701:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3687 00000100:00080000:0.0:1787828887.399230:0:40701:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787828887.399232:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787828887.399235:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000800:00000100:2.0:1787828887.399237:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000020:00000001:0.0:1787828887.399239:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787828887.399242:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787828887.399244:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000020:00000001:0.0:1787828887.399246:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:0.0:1787828887.399249:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787828887.399250:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:2.0:1787828887.399253:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00080000:0.0:1787828887.399255:0:40701:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787828887.399260:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828887.399260:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 02000000:00000001:0.0:1787828887.399262:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:2.0:1787828887.399265:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828887.399267:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787828887.399269:0:2390:0:(events.c:68:request_out_callback()) Process leaving 02000000:00000001:0.0:1787828887.399272:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828887.399276:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:2.0:1787828887.399277:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000008:00100000:0.0:1787828887.399278:0:40701:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1787828887.399283:0:40701:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828887.399284:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4298361600 00000100:00000001:0.0:1787828887.399285:0:40701:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828887.399294:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787828887.399295:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828887.399297:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:2.0:1787828887.399298:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 02000000:00000001:0.0:1787828887.399299:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787828887.399303:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828887.399309:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4298361600 00000100:00000001:0.0:1787828887.399310:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828887.399320:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d270c0 x1874670646775040/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787828887.399320:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 00000100:00000001:2.0:1787828887.399327:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787828887.399336:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828887.399337:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787828887.399340:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828887.399342:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828887.399351:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1787828887.399359:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828887/real 1787828887] req@ffff8d30d4d25a40 x1874670646774656/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828912 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1787828887.399370:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787828887.399371:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1787828887.399375:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1787828887.399377:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787828887.399379:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828887.399381:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828887.399383:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787828887.399387:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d25a40 x1874670646774656/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828912 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787828887.399394:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828887.399397:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787828887.399399:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828887.399401:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828887.399402:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828887.399420:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787828887.399421:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787828887.399423:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828887.399426:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:2.0:1787828887.399428:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1787828887.399431:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d25a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646774656:192.168.203.143@tcp:250:kworker.0 00000100:00000001:2.0:1787828887.399435:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828887.399436:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828887.399438:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787828887.399440:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1787828887.399442:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787828887.399445:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:2.0:1787828887.399447:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:2.0:1787828887.399450:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787828887.399452:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1787828887.399454:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787828887.399455:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828887.399457:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828887.399463:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1787828887.399465:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828887.399467:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787828887.399468:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1787828887.399469:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1787828887.399470:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787828887.399471:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828887.399473:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787828887.399475:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828887.399476:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787828887.399479:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d24f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646774784:192.168.203.143@tcp:38:kworker.0 00000100:00000001:2.0:1787828887.399481:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787828887.399483:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1787828887.399484:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828887.399486:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1787828887.399488:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828887.399492:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787828887.399493:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601340160 : -126234108211456 : ffff8d30d4d24f00) 00000100:00000001:2.0:1787828887.399497:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1787828887.399507:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:2.0:1787828887.399511:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1787828887.399512:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828887.399513:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828887.399515:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1787828887.399517:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828887.399520:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298360342 00000100:00000001:2.0:1787828887.399522:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298361600 00000100:00000001:2.0:1787828887.399524:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298360342 00000100:00000001:2.0:1787828887.399526:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298361600 00000100:00000001:2.0:1787828887.399528:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787828887.399529:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828887.399531:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1787828887.399534:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828887/real 1787828887] req@ffff8d30d4d24f00 x1874670646774784/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828912 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1787828887.399540:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787828887.399540:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1787828887.399542:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1787828887.399543:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787828887.399544:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828887.399545:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828887.399546:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787828887.399549:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d24f00 x1874670646774784/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828912 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787828887.399554:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828887.399556:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787828887.399557:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828887.399558:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828887.399559:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828887.399560:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787828887.399562:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787828887.399563:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828887.399565:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:2.0:1787828887.399567:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1787828887.399569:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d24f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646774784:192.168.203.143@tcp:38:kworker.0 00000100:00000001:2.0:1787828887.399571:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1787828887.399572:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787828887.399573:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828887.399574:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787828887.399576:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828887.399577:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787828887.399580:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d24000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646774912:192.168.203.143@tcp:38:kworker.0 00000100:00000001:2.0:1787828887.399582:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787828887.399583:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1787828887.399585:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828887.399586:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1787828887.399588:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828887.399591:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787828887.399592:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601336320 : -126234108215296 : ffff8d30d4d24000) 00000100:00000001:2.0:1787828887.399595:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1787828887.399601:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:2.0:1787828887.399604:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1787828887.399605:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828887.399606:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828887.399625:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1787828887.399627:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828887.399659:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298360343 00000100:00000001:2.0:1787828887.399662:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298361600 00000100:00000001:2.0:1787828887.399664:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298360343 00000100:00000001:2.0:1787828887.399666:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298361600 00000100:00000001:2.0:1787828887.399668:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787828887.399670:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828887.399671:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1787828887.399691:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828887/real 1787828887] req@ffff8d30d4d24000 x1874670646774912/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828912 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1787828887.399696:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787828887.399697:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1787828887.399700:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1787828887.399701:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787828887.399702:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828887.399703:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828887.399705:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787828887.399707:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d24000 x1874670646774912/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828912 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787828887.399717:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828887.399719:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787828887.399721:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828887.399722:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828887.399723:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828887.399724:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787828887.399725:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787828887.399727:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828887.399729:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:2.0:1787828887.399730:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1787828887.399732:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d24000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646774912:192.168.203.143@tcp:38:kworker.0 00000100:00000001:2.0:1787828887.399735:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1787828887.399736:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787828887.399754:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828887.399756:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787828887.399757:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828887.399759:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787828887.399762:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d270c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646775040:192.168.203.143@tcp:8:kworker.0 00000100:00000001:2.0:1787828887.399764:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787828887.399765:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1787828887.399767:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828887.399768:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1787828887.399771:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828887.399776:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787828887.399777:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601348800 : -126234108202816 : ffff8d30d4d270c0) 00000100:00000001:2.0:1787828887.399779:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1787828887.399789:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:2.0:1787828887.399793:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1787828887.399805:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828887.399806:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828887.399808:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1787828887.399810:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828887.399812:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298360356 00000100:00000001:2.0:1787828887.399814:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298361600 00000100:00000001:2.0:1787828887.399816:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298360356 00000100:00000001:2.0:1787828887.399818:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298361600 00000100:00000001:2.0:1787828887.399820:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787828887.399821:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828887.399822:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1787828887.399825:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828887/real 1787828887] req@ffff8d30d4d270c0 x1874670646775040/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828912 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1787828887.399831:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787828887.399832:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1787828887.399833:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1787828887.399834:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787828887.399835:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828887.399837:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828887.399838:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787828887.399840:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d270c0 x1874670646775040/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828912 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787828887.399846:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828887.399848:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787828887.399850:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828887.399851:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828887.399852:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828887.399853:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787828887.399854:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787828887.399856:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828887.399858:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:2.0:1787828887.399860:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1787828887.399863:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d270c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646775040:192.168.203.143@tcp:8:kworker.0 00000100:00000001:2.0:1787828887.399866:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828887.399867:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828887.399868:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787828887.399870:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1787828887.399871:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787828887.399872:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:2.0:1787828887.399873:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:2.0:1787828887.399875:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787828887.399876:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1787828887.399879:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787828887.399897:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828887.399900:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828887.399901:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787828887.399902:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1787828887.399903:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787828887.399904:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:2.0:1787828887.399905:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:2.0:1787828887.399907:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787828887.399908:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1787828887.399909:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787828887.399910:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828887.399912:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828887.399912:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787828887.399913:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1787828887.399915:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787828887.399915:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:2.0:1787828887.399916:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:2.0:1787828887.399917:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787828887.399918:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1787828887.399921:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787828887.399922:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828887.399923:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828887.399926:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1787828887.399927:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828887.399928:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787828887.399930:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828888.294679:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828888.294685:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828888.294773:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828888.294776:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828888.294780:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828889.318741:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828889.318747:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828889.318842:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828889.318847:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828889.318851:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828890.342708:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828890.342713:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828890.342766:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828890.342769:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828890.342772:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828891.366739:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828891.366745:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828891.366815:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828891.366819:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828891.366822:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828892.390702:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828892.390719:0:815:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1787828892.390722:0:815:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1787828892.390724:0:815:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828892.390728:0:815:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1787828892.390730:0:815:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828892.390732:0:815:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828892.390734:0:815:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828892.390815:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828892.390820:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828892.390825:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787828892.518741:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828892.518754:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1787828892.518757:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1787828892.518761:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787828892.518767:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1787828892.518776:0:468:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 3693 00000100:00080000:1.0:1787828892.518789:0:468:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1787828892.518793:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787828892.518796:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:1.0:1787828892.518800:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:1.0:1787828892.518803:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:1.0:1787828892.518814:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:1.0:1787828892.518816:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:1.0:1787828892.518818:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787828892.518819:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:1.0:1787828892.518824:0:468:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:1.0:1787828892.518826:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828892.518829:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1787828892.518833:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787828892.518836:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828892.518838:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:1.0:1787828892.518842:0:468:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828892.518872:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1787828892.518876:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828892.518882:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1787828892.518885:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1787828892.518891:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828892.518908:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787828892.518921:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d13ec3c0 x1874670646775168/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787828892.518942:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787828892.518948:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828892.518970:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1787828892.518973:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828892.518976:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1787828892.518977:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1787828892.518979:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787828892.518981:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1787828892.518983:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3693 00000100:00080000:1.0:1787828892.518987:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1787828892.518989:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787828892.518990:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:1.0:1787828892.518992:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:1.0:1787828892.518993:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:1.0:1787828892.518995:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000020:00000001:1.0:1787828892.518997:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:1.0:1787828892.518998:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787828892.518999:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:1.0:1787828892.519002:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:1.0:1787828892.519003:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828892.519004:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1787828892.519009:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787828892.519011:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828892.519013:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:1.0:1787828892.519018:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1787828892.519020:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828892.519022:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828892.519026:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1787828892.519033:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828892.519035:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1787828892.519037:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:2.0:1787828892.519037:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:1.0:1787828892.519039:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828892.519044:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1787828892.519046:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828892.519049:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00100000:1.0:1787828892.519053:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d13ecb40 x1874670646775296/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787828892.519054:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787828892.519057:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828892.519064:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787828892.519067:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828892.519069:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:1.0:1787828892.519070:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828892.519073:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00080000:1.0:1787828892.519074:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 02000000:00000001:2.0:1787828892.519077:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1787828892.519079:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:1.0:1787828892.519082:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1787828892.519085:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1787828892.519086:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d13ec3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646775168:192.168.203.143@tcp:250:kworker.0 00000100:00000001:1.0:1787828892.519088:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1787828892.519093:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3693 00000100:00000001:2.0:1787828892.519094:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00080000:1.0:1787828892.519097:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1787828892.519099:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787828892.519100:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 02000000:00000001:2.0:1787828892.519100:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000020:00000001:1.0:1787828892.519102:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:1.0:1787828892.519105:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 02000000:00000001:2.0:1787828892.519105:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787828892.519109:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 02000000:00000001:2.0:1787828892.519111:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000020:00000001:1.0:1787828892.519114:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:1.0:1787828892.519116:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787828892.519117:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 02000000:00000001:2.0:1787828892.519118:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1787828892.519120:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:1.0:1787828892.519124:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828892.519125:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1787828892.519127:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787828892.519129:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828892.519130:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:1.0:1787828892.519132:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1787828892.519133:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828892.519135:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828892.519138:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1787828892.519139:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828892.519140:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1787828892.519140:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1787828892.519142:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828892.519145:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828892.519146:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00100000:1.0:1787828892.519150:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d13ec000 x1874670646775424/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787828892.519151:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839541339072 : -126234168212544 : ffff8d30d13ec3c0) 00000100:00000001:2.0:1787828892.519160:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:1.0:1787828892.519161:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787828892.519165:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828892.519168:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1787828892.519169:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828892.519172:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1787828892.519173:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1787828892.519174:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787828892.519175:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1787828892.519177:0:468:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3693 00000100:00080000:1.0:1787828892.519179:0:468:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1787828892.519180:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787828892.519181:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:1.0:1787828892.519182:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:1.0:1787828892.519183:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:1.0:1787828892.519184:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000020:00000001:1.0:1787828892.519186:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:1.0:1787828892.519187:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787828892.519188:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:1.0:1787828892.519195:0:468:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:1.0:1787828892.519198:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828892.519199:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1787828892.519202:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787828892.519203:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828892.519205:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:1.0:1787828892.519207:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000008:00000001:1.0:1787828892.519209:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828892.519210:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828892.519213:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1787828892.519215:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828892.519216:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1787828892.519217:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1787828892.519219:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828892.519222:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787828892.519225:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d13ec780 x1874670646775552/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787828892.519230:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828892.519233:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787828892.519235:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828892.519237:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4298366720 00000100:00000001:1.0:1787828892.519238:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828892.519250:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 00000100:00000001:2.0:1787828892.519258:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4298366720 00000100:00000001:2.0:1787828892.519262:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 00000100:00000001:2.0:1787828892.519265:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787828892.519269:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828892.519273:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828892.519276:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828892.519285:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1787828892.519286:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:2.0:1787828892.519288:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787828892.519290:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1787828892.519291:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1787828892.519292:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787828892.519293:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828892.519295:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787828892.519296:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828892.519297:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787828892.519301:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d13ecb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646775296:192.168.203.143@tcp:38:kworker.0 00000100:00000001:2.0:1787828892.519304:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787828892.519305:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1787828892.519307:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828892.519309:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1787828892.519312:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828892.519317:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787828892.519318:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839541340992 : -126234168210624 : ffff8d30d13ecb40) 00000100:00000001:2.0:1787828892.519323:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:2.0:1787828892.519338:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828892.519342:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298365462 00000100:00000001:2.0:1787828892.519351:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298366720 00000100:00000001:2.0:1787828892.519354:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298365462 00000100:00000001:2.0:1787828892.519357:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298366720 00000100:00000001:2.0:1787828892.519359:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787828892.519360:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828892.519362:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1787828892.519363:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787828892.519364:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828892.519365:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787828892.519366:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828892.519368:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787828892.519371:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d13ec000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646775424:192.168.203.143@tcp:38:kworker.0 00000100:00000001:2.0:1787828892.519374:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787828892.519375:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1787828892.519377:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828892.519378:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1787828892.519382:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828892.519387:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787828892.519389:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839541338112 : -126234168213504 : ffff8d30d13ec000) 00000100:00000001:2.0:1787828892.519393:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:2.0:1787828892.519408:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828892.519412:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298365462 00000100:00000001:2.0:1787828892.519415:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298366720 00000100:00000001:2.0:1787828892.519417:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298365462 00000100:00000001:2.0:1787828892.519420:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298366720 00000100:00000001:2.0:1787828892.519423:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787828892.519425:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828892.519427:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1787828892.519429:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787828892.519430:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828892.519433:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787828892.519436:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828892.519439:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787828892.519443:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d13ec780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646775552:192.168.203.143@tcp:8:kworker.0 00000100:00000001:2.0:1787828892.519447:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787828892.519449:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1787828892.519451:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828892.519455:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1787828892.519460:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828892.519466:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787828892.519468:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839541340032 : -126234168211584 : ffff8d30d13ec780) 00000100:00000001:2.0:1787828892.519471:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:2.0:1787828892.519483:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828892.519486:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298365475 00000100:00000001:2.0:1787828892.519489:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298366720 00000100:00000001:2.0:1787828892.519491:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298365475 00000100:00000001:2.0:1787828892.519493:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298366720 00000100:00000001:2.0:1787828892.519496:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787828892.519497:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828892.519499:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828892.519502:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828892.519504:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1787828892.519506:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:2.0:1787828892.519507:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787828892.519508:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1787828892.519510:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828892.519513:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828893.414673:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828893.414678:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828893.414733:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828893.414737:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828893.414741:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828894.438667:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828894.438672:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828894.438732:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828894.438737:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828894.438741:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828895.462786:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828895.462793:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828895.462913:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828895.462922:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828895.462927:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:3.0:1787828895.590960:0:2380:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.143:988 00000400:00000100:3.0:1787828895.591039:0:2380:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.143@tcp at host 192.168.203.143 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:3.0:1787828895.591065:0:2380:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.143@tcp 00000100:00000001:3.0:1787828895.591085:0:2380:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787828895.591118:0:2380:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787828895.591121:0:2380:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828895.591124:0:2380:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:3.0:1787828895.591136:0:2380:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.143@tcp 00000100:00000001:3.0:1787828895.591143:0:2380:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787828895.591145:0:2380:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787828895.591147:0:2380:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828895.591149:0:2380:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:3.0:1787828895.591154:0:2380:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.143@tcp 00000100:00000001:3.0:1787828895.591162:0:2380:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787828895.591165:0:2380:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787828895.591166:0:2380:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828895.591168:0:2380:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:3.0:1787828895.591173:0:2380:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.143@tcp 00000100:00000001:3.0:1787828895.591180:0:2380:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787828895.591183:0:2380:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787828895.591184:0:2380:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828895.591186:0:2380:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1787828895.591265:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787828895.591271:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1787828895.591275:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1787828895.591289:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828892/real 1787828895] req@ffff8d30d13ecb40 x1874670646775296/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828917 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1787828895.591308:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787828895.591311:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1787828895.591318:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1787828895.591322:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787828895.591325:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828895.591329:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828895.591337:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787828895.591343:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d13ecb40 x1874670646775296/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828917 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787828895.591350:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828895.591356:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787828895.591360:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828895.591362:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828895.591363:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828895.591366:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787828895.591370:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787828895.591374:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828895.591380:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:2.0:1787828895.591384:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1787828895.591389:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d13ecb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646775296:192.168.203.143@tcp:38:kworker.0 00000100:00000001:2.0:1787828895.591395:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1787828895.591400:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828892/real 1787828895] req@ffff8d30d13ec000 x1874670646775424/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828917 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1787828895.591410:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787828895.591411:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1787828895.591415:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1787828895.591417:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787828895.591418:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828895.591420:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828895.591422:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787828895.591425:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d13ec000 x1874670646775424/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828917 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787828895.591432:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828895.591435:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787828895.591437:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828895.591439:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828895.591441:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828895.591442:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787828895.591444:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787828895.591448:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828895.591453:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:2.0:1787828895.591456:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1787828895.591459:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d13ec000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646775424:192.168.203.143@tcp:38:kworker.0 00000100:00000001:2.0:1787828895.591464:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1787828895.591468:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828892/real 1787828895] req@ffff8d30d13ec780 x1874670646775552/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828917 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1787828895.591478:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787828895.591479:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1787828895.591483:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1787828895.591485:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787828895.591486:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828895.591488:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828895.591490:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787828895.591494:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d13ec780 x1874670646775552/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828917 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787828895.591500:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828895.591502:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787828895.591504:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828895.591506:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828895.591508:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828895.591509:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787828895.591512:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787828895.591514:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828895.591518:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:2.0:1787828895.591521:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1787828895.591525:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d13ec780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646775552:192.168.203.143@tcp:8:kworker.0 00000100:00000001:2.0:1787828895.591529:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1787828895.591534:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828892/real 1787828895] req@ffff8d30d13ec3c0 x1874670646775168/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828917 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1787828895.591544:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787828895.591545:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1787828895.591549:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1787828895.591550:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787828895.591552:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828895.591554:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828895.591556:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787828895.591560:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d13ec3c0 x1874670646775168/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828917 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787828895.591566:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828895.591568:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787828895.591570:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828895.591572:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828895.591573:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828895.591575:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787828895.591577:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787828895.591579:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828895.591583:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:2.0:1787828895.591585:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1787828895.591588:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d13ec3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646775168:192.168.203.143@tcp:250:kworker.0 00000100:00000001:2.0:1787828895.591593:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828895.591595:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828895.591597:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787828895.591636:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1787828895.591658:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787828895.591661:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:2.0:1787828895.591665:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:2.0:1787828895.591670:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787828895.591673:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1787828895.591677:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787828895.591679:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828895.591681:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828895.591683:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787828895.591684:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1787828895.591687:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787828895.591688:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:2.0:1787828895.591689:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:2.0:1787828895.591691:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787828895.591692:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1787828895.591694:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787828895.591695:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828895.591697:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828895.591698:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787828895.591700:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1787828895.591702:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787828895.591703:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:2.0:1787828895.591704:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:2.0:1787828895.591706:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787828895.591724:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1787828895.591725:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787828895.591727:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828895.591728:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828895.591730:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787828895.591731:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1787828895.591733:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787828895.591734:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:2.0:1787828895.591736:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:2.0:1787828895.591737:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787828895.591738:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1787828895.591740:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787828895.591741:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828895.591743:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828895.591756:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1787828895.591759:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828895.591761:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787828895.591762:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828896.486748:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828896.486767:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828896.486863:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828896.486887:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828896.486891:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828897.510706:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828897.510715:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828897.510780:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828897.510786:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828897.510790:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787828897.638839:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828897.638854:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1787828897.638858:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1787828897.638862:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787828897.638868:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1787828897.638877:0:468:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 3698 00000100:00080000:1.0:1787828897.638892:0:468:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1787828897.638897:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787828897.638900:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:1.0:1787828897.638906:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:1.0:1787828897.638909:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:1.0:1787828897.638914:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:1.0:1787828897.638918:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:1.0:1787828897.638922:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787828897.638924:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:1.0:1787828897.638931:0:468:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:1.0:1787828897.638935:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828897.638938:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1787828897.638944:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787828897.638948:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828897.638950:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:1.0:1787828897.638953:0:468:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828897.638963:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1787828897.638965:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828897.638970:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1787828897.638974:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1787828897.638979:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828897.638993:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787828897.639005:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d13ee940 x1874670646775680/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787828897.639017:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787828897.639021:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828897.639038:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1787828897.639041:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828897.639049:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1787828897.639051:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1787828897.639052:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787828897.639055:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1787828897.639058:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3698 00000100:00080000:1.0:1787828897.639063:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1787828897.639065:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787828897.639068:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:1.0:1787828897.639072:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:1.0:1787828897.639074:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:1.0:1787828897.639078:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000020:00000001:1.0:1787828897.639080:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:1.0:1787828897.639083:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787828897.639086:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:1.0:1787828897.639090:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:1.0:1787828897.639092:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828897.639094:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1787828897.639104:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787828897.639108:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828897.639110:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:1.0:1787828897.639116:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1787828897.639119:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828897.639122:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828897.639126:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:1.0:1787828897.639128:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1787828897.639136:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828897.639140:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1787828897.639145:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:0.0:1787828897.639147:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 02000000:00000001:1.0:1787828897.639152:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828897.639158:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787828897.639161:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787828897.639170:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d13ec3c0 x1874670646775808/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787828897.639176:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787828897.639178:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787828897.639189:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828897.639194:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828897.639199:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1787828897.639210:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 02000000:00000001:0.0:1787828897.639214:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000100:00080000:1.0:1787828897.639215:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:1.0:1787828897.639222:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1787828897.639227:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 02000000:00000001:0.0:1787828897.639229:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787828897.639231:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1787828897.639242:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3698 02000000:00000001:0.0:1787828897.639243:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1787828897.639250:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1787828897.639254:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787828897.639256:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:1.0:1787828897.639258:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:1.0:1787828897.639259:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:1.0:1787828897.639261:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000020:00000001:1.0:1787828897.639264:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:1.0:1787828897.639268:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787828897.639270:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00100000:0.0:1787828897.639271:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d13ee940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646775680:192.168.203.143@tcp:250:kworker.0 00000100:00080000:1.0:1787828897.639280:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:1.0:1787828897.639291:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828897.639301:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:0.0:1787828897.639307:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787828897.639307:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787828897.639317:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828897.639322:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:1.0:1787828897.639322:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 02000000:00000001:0.0:1787828897.639334:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000008:00100000:1.0:1787828897.639339:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 02000000:00000001:0.0:1787828897.639344:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000008:00000001:1.0:1787828897.639350:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828897.639358:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828897.639365:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1787828897.639370:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828897.639373:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1787828897.639375:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1787828897.639377:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828897.639384:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787828897.639391:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d13ec780 x1874670646775936/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787828897.639402:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787828897.639405:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828897.639408:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1787828897.639413:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 02000000:00000001:0.0:1787828897.639415:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1787828897.639429:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1787828897.639431:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1787828897.639435:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787828897.639439:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1787828897.639443:0:468:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3698 00000100:00080000:1.0:1787828897.639449:0:468:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1787828897.639453:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787828897.639454:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:1.0:1787828897.639459:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:0.0:1787828897.639461:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000020:00000001:1.0:1787828897.639464:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:0.0:1787828897.639469:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839541348672 : -126234168202944 : ffff8d30d13ee940) 00000020:00000001:1.0:1787828897.639472:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000020:00000001:1.0:1787828897.639484:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:0.0:1787828897.639487:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:1.0:1787828897.639490:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787828897.639495:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:1.0:1787828897.639501:0:468:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:1.0:1787828897.639507:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828897.639511:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1787828897.639517:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787828897.639520:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828897.639523:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:1.0:1787828897.639526:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000008:00000001:1.0:1787828897.639529:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828897.639530:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828897.639539:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1787828897.639541:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828897.639544:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1787828897.639545:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1787828897.639548:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828897.639556:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:0.0:1787828897.639572:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00100000:1.0:1787828897.639575:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d13ec000 x1874670646776064/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787828897.639589:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787828897.639593:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828897.639596:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787828897.639597:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828897.639655:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828897.639666:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828897.639672:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787828897.639686:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828897.639696:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4298371841 00000100:00000001:0.0:1787828897.639716:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1787828897.639728:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4298371841 00000100:00000001:0.0:1787828897.639737:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1787828897.639744:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787828897.639750:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828897.639754:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787828897.639770:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828897/real 1787828897] req@ffff8d30d13ee940 x1874670646775680/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828922 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787828897.639802:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787828897.639806:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787828897.639840:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787828897.639846:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787828897.639850:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828897.639855:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828897.639859:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787828897.639869:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d13ee940 x1874670646775680/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828922 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787828897.639885:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828897.639892:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787828897.639897:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828897.639901:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828897.639903:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828897.639908:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787828897.639912:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787828897.639918:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828897.639926:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787828897.639931:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787828897.639938:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d13ee940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646775680:192.168.203.143@tcp:250:kworker.0 00000100:00000001:0.0:1787828897.639947:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828897.639951:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828897.639954:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787828897.639959:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787828897.639965:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787828897.639969:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787828897.639973:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787828897.639981:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787828897.639984:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787828897.639991:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787828897.639993:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828897.639998:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828897.640010:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787828897.640014:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828897.640019:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787828897.640023:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787828897.640026:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1787828897.640030:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787828897.640033:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828897.640038:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787828897.640042:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787828897.640047:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828897.640054:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d13ec3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646775808:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787828897.640061:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787828897.640066:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787828897.640069:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828897.640073:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1787828897.640080:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828897.640093:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787828897.640096:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839541339072 : -126234168212544 : ffff8d30d13ec3c0) 00000100:00000001:0.0:1787828897.640105:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1787828897.640136:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:0.0:1787828897.640146:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1787828897.640152:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828897.640155:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828897.640158:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787828897.640164:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828897.640172:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298370583 00000100:00000001:0.0:1787828897.640178:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298371841 00000100:00000001:0.0:1787828897.640183:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298370583 00000100:00000001:0.0:1787828897.640186:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298371841 00000100:00000001:0.0:1787828897.640190:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787828897.640195:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828897.640198:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787828897.640208:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828897/real 1787828897] req@ffff8d30d13ec3c0 x1874670646775808/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828922 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787828897.640227:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787828897.640229:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787828897.640235:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787828897.640239:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787828897.640242:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828897.640246:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828897.640249:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787828897.640257:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d13ec3c0 x1874670646775808/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828922 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787828897.640273:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828897.640278:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787828897.640283:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828897.640286:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828897.640289:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828897.640294:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787828897.640298:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787828897.640303:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828897.640311:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787828897.640316:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787828897.640323:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d13ec3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646775808:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787828897.640331:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1787828897.640335:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787828897.640338:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828897.640343:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787828897.640347:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787828897.640351:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828897.640357:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d13ec780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646775936:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787828897.640364:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787828897.640368:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787828897.640371:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828897.640375:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1787828897.640381:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828897.640392:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787828897.640395:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839541340032 : -126234168211584 : ffff8d30d13ec780) 00000100:00000001:0.0:1787828897.640402:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1787828897.640429:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:0.0:1787828897.640439:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1787828897.640444:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828897.640447:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828897.640451:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787828897.640457:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828897.640464:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298370583 00000100:00000001:0.0:1787828897.640469:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298371841 00000100:00000001:0.0:1787828897.640474:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298370583 00000100:00000001:0.0:1787828897.640479:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298371841 00000100:00000001:0.0:1787828897.640484:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787828897.640488:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828897.640492:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787828897.640517:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828897/real 1787828897] req@ffff8d30d13ec780 x1874670646775936/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828922 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787828897.640531:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787828897.640534:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787828897.640538:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787828897.640541:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787828897.640544:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828897.640546:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828897.640549:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787828897.640556:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d13ec780 x1874670646775936/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828922 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787828897.640580:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828897.640584:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787828897.640590:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828897.640593:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828897.640595:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828897.640598:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787828897.640641:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787828897.640646:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828897.640654:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787828897.640658:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787828897.640664:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d13ec780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646775936:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787828897.640671:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1787828897.640674:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787828897.640677:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828897.640681:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787828897.640684:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787828897.640688:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828897.640693:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d13ec000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646776064:192.168.203.143@tcp:8:kworker.0 00000100:00000001:0.0:1787828897.640697:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787828897.640700:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787828897.640702:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828897.640704:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1787828897.640709:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828897.640719:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787828897.640722:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839541338112 : -126234168213504 : ffff8d30d13ec000) 00000100:00000001:0.0:1787828897.640728:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1787828897.640752:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:0.0:1787828897.640759:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1787828897.640763:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828897.640767:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828897.640770:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787828897.640775:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828897.640781:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298370597 00000100:00000001:0.0:1787828897.640786:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298371841 00000100:00000001:0.0:1787828897.640790:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298370597 00000100:00000001:0.0:1787828897.640794:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298371841 00000100:00000001:0.0:1787828897.640798:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787828897.640801:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828897.640805:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787828897.640833:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828897/real 1787828897] req@ffff8d30d13ec000 x1874670646776064/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828922 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787828897.640849:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787828897.640851:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787828897.640856:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787828897.640860:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787828897.640862:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828897.640865:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828897.640868:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787828897.640874:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d13ec000 x1874670646776064/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828922 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787828897.640887:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828897.640891:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787828897.640895:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828897.640898:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828897.640900:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828897.640904:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787828897.640907:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787828897.640912:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828897.640917:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787828897.640921:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787828897.640927:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d13ec000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646776064:192.168.203.143@tcp:8:kworker.0 00000100:00000001:0.0:1787828897.640934:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828897.640937:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828897.640939:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787828897.640942:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787828897.640946:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787828897.640949:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787828897.640951:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787828897.640955:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787828897.640957:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787828897.640961:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787828897.640963:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828897.640966:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828897.640968:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787828897.640970:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787828897.640973:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787828897.640975:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787828897.640977:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787828897.640981:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787828897.640983:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787828897.640986:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787828897.640988:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828897.640991:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828897.640993:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787828897.640995:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787828897.640998:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787828897.641000:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787828897.641002:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787828897.641005:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787828897.641008:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787828897.641010:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787828897.641012:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828897.641016:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828897.641023:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787828897.641026:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828897.641029:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787828897.641032:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828898.534707:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828898.534713:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828898.534769:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828898.534773:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828898.534776:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828899.558719:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828899.558725:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828899.558793:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828899.558815:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828899.558819:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828900.582693:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828900.582698:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828900.582757:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828900.582760:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828900.582762:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828901.606758:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828901.606764:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828901.606862:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828901.606866:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828901.606868:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828902.630703:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828902.630710:0:815:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1787828902.630712:0:815:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1787828902.630715:0:815:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828902.630720:0:815:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1787828902.630738:0:815:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828902.630741:0:815:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828902.630743:0:815:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828902.630847:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828902.630850:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828902.630853:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787828902.758729:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828902.758739:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1787828902.758743:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787828902.758746:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787828902.758751:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787828902.758758:0:40701:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 3703 00000100:00080000:0.0:1787828902.758771:0:40701:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787828902.758787:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787828902.758789:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787828902.758793:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787828902.758796:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787828902.758799:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:0.0:1787828902.758801:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:0.0:1787828902.758803:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787828902.758804:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:0.0:1787828902.758808:0:40701:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787828902.758810:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828902.758812:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787828902.758815:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787828902.758818:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828902.758820:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1787828902.758822:0:40701:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828902.758848:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787828902.758849:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828902.758854:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787828902.758856:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787828902.758861:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828902.758875:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828902.758884:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d27c00 x1874670646776192/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787828902.758893:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787828902.758897:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828902.758916:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787828902.758918:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828902.758921:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1787828902.758922:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787828902.758923:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787828902.758926:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787828902.758928:0:40701:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3703 00000100:00080000:0.0:1787828902.758931:0:40701:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787828902.758932:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787828902.758934:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787828902.758935:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787828902.758936:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787828902.758938:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000020:00000001:0.0:1787828902.758939:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:0.0:1787828902.758941:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787828902.758942:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:0.0:1787828902.758943:0:40701:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787828902.758945:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828902.758946:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787828902.758951:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787828902.758953:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828902.758954:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1787828902.758959:0:40701:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1787828902.758961:0:40701:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828902.758963:0:40701:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828902.758965:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787828902.758966:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828902.758967:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787828902.758968:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787828902.758971:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828902.758976:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828902.758981:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d252c0 x1874670646776320/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787828902.758988:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787828902.758989:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828902.758991:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787828902.758994:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828902.758999:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1787828902.759001:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787828902.759004:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787828902.759013:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787828902.759013:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1787828902.759019:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1787828902.759026:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:2.0:1787828902.759027:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787828902.759034:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787828902.759041:0:40701:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3703 02000000:00000001:2.0:1787828902.759044:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000100:00080000:0.0:1787828902.759055:0:40701:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 02000000:00000001:2.0:1787828902.759056:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828902.759070:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828902.759071:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787828902.759080:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00100000:2.0:1787828902.759087:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d27c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646776192:192.168.203.143@tcp:250:kworker.0 00000020:00000001:0.0:1787828902.759089:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787828902.759093:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787828902.759099:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:2.0:1787828902.759102:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000020:00000001:0.0:1787828902.759106:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:0.0:1787828902.759109:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 02000000:00000001:2.0:1787828902.759110:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:0.0:1787828902.759111:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 02000000:00000001:2.0:1787828902.759116:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828902.759121:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00080000:0.0:1787828902.759122:0:40701:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787828902.759126:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828902.759127:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787828902.759131:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828902.759131:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828902.759136:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828902.759139:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1787828902.759141:0:40701:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1787828902.759144:0:40701:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828902.759146:0:40701:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828902.759150:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:2.0:1787828902.759151:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 02000000:00000001:0.0:1787828902.759152:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828902.759155:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601351680 : -126234108199936 : ffff8d30d4d27c00) 00000100:00000001:0.0:1787828902.759157:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787828902.759161:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:2.0:1787828902.759163:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 02000000:00000001:0.0:1787828902.759165:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828902.759173:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828902.759180:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d25680 x1874670646776448/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787828902.759187:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787828902.759189:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828902.759194:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787828902.759196:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828902.759200:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1787828902.759201:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787828902.759205:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:2.0:1787828902.759207:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:0.0:1787828902.759210:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:2.0:1787828902.759222:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00080000:0.0:1787828902.759223:0:40701:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3703 00000100:00080000:0.0:1787828902.759236:0:40701:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787828902.759236:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828902.759242:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828902.759243:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828902.759252:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:2.0:1787828902.759252:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000020:00000001:0.0:1787828902.759261:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787828902.759263:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:2.0:1787828902.759264:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787828902.759270:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:2.0:1787828902.759273:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4298376960 00000020:00000001:0.0:1787828902.759274:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:0.0:1787828902.759279:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787828902.759280:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:0.0:1787828902.759285:0:40701:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787828902.759285:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1787828902.759287:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828902.759293:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787828902.759296:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787828902.759298:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828902.759298:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4298376960 00000100:00000001:0.0:1787828902.759305:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1787828902.759309:0:40701:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00000001:2.0:1787828902.759310:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 00000008:00000001:0.0:1787828902.759318:0:40701:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828902.759319:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787828902.759332:0:40701:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828902.759332:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828902.759346:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 02000000:00000001:0.0:1787828902.759348:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787828902.759350:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828902.759354:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787828902.759356:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00100000:2.0:1787828902.759357:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828902/real 1787828902] req@ffff8d30d4d27c00 x1874670646776192/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828927 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:0.0:1787828902.759359:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828902.759369:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828902.759383:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d26940 x1874670646776576/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000001:2.0:1787828902.759383:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787828902.759386:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1787828902.759392:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1787828902.759400:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787828902.759403:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787828902.759405:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828902.759409:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828902.759411:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828902.759413:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828902.759414:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787828902.759434:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d27c00 x1874670646776192/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828927 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787828902.759441:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828902.759444:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787828902.759447:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828902.759450:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828902.759452:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828902.759455:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787828902.759457:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787828902.759461:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828902.759465:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:2.0:1787828902.759467:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1787828902.759472:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d27c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646776192:192.168.203.143@tcp:250:kworker.0 00000100:00000001:2.0:1787828902.759477:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1787828902.759479:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787828902.759481:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828902.759484:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787828902.759487:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828902.759489:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787828902.759493:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d252c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646776320:192.168.203.143@tcp:38:kworker.0 00000100:00000001:2.0:1787828902.759498:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787828902.759501:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1787828902.759502:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828902.759505:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1787828902.759509:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828902.759515:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787828902.759517:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601341120 : -126234108210496 : ffff8d30d4d252c0) 00000100:00000001:2.0:1787828902.759522:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1787828902.759538:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:2.0:1787828902.759543:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1787828902.759545:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828902.759546:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828902.759547:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1787828902.759549:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828902.759553:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298375702 00000100:00000001:2.0:1787828902.759556:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298376960 00000100:00000001:2.0:1787828902.759557:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298375702 00000100:00000001:2.0:1787828902.759559:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298376960 00000100:00000001:2.0:1787828902.759561:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787828902.759562:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828902.759564:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1787828902.759567:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828902/real 1787828902] req@ffff8d30d4d252c0 x1874670646776320/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828927 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1787828902.759574:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787828902.759575:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1787828902.759577:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1787828902.759578:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787828902.759579:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828902.759580:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828902.759581:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787828902.759585:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d252c0 x1874670646776320/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828927 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787828902.759591:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828902.759592:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787828902.759594:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828902.759595:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828902.759596:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828902.759597:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787828902.759640:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787828902.759643:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828902.759646:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:2.0:1787828902.759647:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1787828902.759650:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d252c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646776320:192.168.203.143@tcp:38:kworker.0 00000100:00000001:2.0:1787828902.759653:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828902.759655:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828902.759656:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787828902.759658:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1787828902.759661:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787828902.759664:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:2.0:1787828902.759666:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:2.0:1787828902.759670:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787828902.759671:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1787828902.759675:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787828902.759676:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828902.759678:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828902.759678:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787828902.759680:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1787828902.759681:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787828902.759682:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:2.0:1787828902.759683:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:2.0:1787828902.759684:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787828902.759685:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1787828902.759687:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787828902.759688:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828902.759690:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828902.759697:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1787828902.759698:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828902.759700:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787828902.759702:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1787828902.759703:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1787828902.759704:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787828902.759706:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828902.759707:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787828902.759709:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828902.759710:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787828902.759713:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d25680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646776448:192.168.203.143@tcp:38:kworker.0 00000100:00000001:2.0:1787828902.759715:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787828902.759717:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1787828902.759719:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828902.759720:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1787828902.759722:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828902.759727:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787828902.759728:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601342080 : -126234108209536 : ffff8d30d4d25680) 00000100:00000001:2.0:1787828902.759730:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1787828902.759739:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:2.0:1787828902.759743:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1787828902.759745:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828902.759746:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828902.759747:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1787828902.759749:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828902.759752:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298375703 00000100:00000001:2.0:1787828902.759754:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298376960 00000100:00000001:2.0:1787828902.759756:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298375703 00000100:00000001:2.0:1787828902.759758:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298376960 00000100:00000001:2.0:1787828902.759759:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787828902.759761:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828902.759762:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1787828902.759765:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828902/real 1787828902] req@ffff8d30d4d25680 x1874670646776448/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828927 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1787828902.759797:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787828902.759798:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1787828902.759801:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1787828902.759802:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787828902.759803:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828902.759805:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828902.759806:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787828902.759810:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d25680 x1874670646776448/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828927 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787828902.759815:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828902.759817:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787828902.759819:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828902.759820:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828902.759821:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828902.759822:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787828902.759823:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787828902.759825:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828902.759827:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:2.0:1787828902.759829:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1787828902.759831:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d25680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646776448:192.168.203.143@tcp:38:kworker.0 00000100:00000001:2.0:1787828902.759834:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1787828902.759836:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787828902.759837:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828902.759839:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787828902.759840:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828902.759842:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787828902.759844:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d26940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646776576:192.168.203.143@tcp:8:kworker.0 00000100:00000001:2.0:1787828902.759846:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787828902.759848:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1787828902.759850:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828902.759851:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1787828902.759854:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828902.759858:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787828902.759859:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601346880 : -126234108204736 : ffff8d30d4d26940) 00000100:00000001:2.0:1787828902.759862:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1787828902.759872:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:2.0:1787828902.759875:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1787828902.759877:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828902.759878:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828902.759879:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1787828902.759881:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828902.759884:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298375716 00000100:00000001:2.0:1787828902.759886:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298376960 00000100:00000001:2.0:1787828902.759887:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298375716 00000100:00000001:2.0:1787828902.759889:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298376960 00000100:00000001:2.0:1787828902.759891:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787828902.759892:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828902.759893:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1787828902.759896:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828902/real 1787828902] req@ffff8d30d4d26940 x1874670646776576/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828927 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1787828902.759901:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787828902.759902:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1787828902.759904:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1787828902.759905:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787828902.759906:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828902.759907:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828902.759908:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787828902.759910:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d26940 x1874670646776576/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828927 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787828902.759915:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828902.759917:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787828902.759919:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828902.759921:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828902.759922:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828902.759923:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787828902.759924:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787828902.759926:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828902.759928:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:2.0:1787828902.759930:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1787828902.759932:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d26940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646776576:192.168.203.143@tcp:8:kworker.0 00000100:00000001:2.0:1787828902.759934:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828902.759936:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828902.759937:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787828902.759938:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1787828902.759939:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787828902.759940:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:2.0:1787828902.759941:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:2.0:1787828902.759943:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787828902.759944:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1787828902.759945:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787828902.759946:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828902.759947:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828902.759948:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787828902.759949:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1787828902.759950:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787828902.759951:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:2.0:1787828902.759951:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:2.0:1787828902.759952:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787828902.759953:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1787828902.759956:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787828902.759957:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828902.759958:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828902.759960:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1787828902.759961:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828902.759963:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787828902.759965:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828903.654797:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828903.654813:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828903.655002:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828903.655006:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828903.655008:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828904.678800:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828904.678835:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828904.678905:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828904.678910:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828904.678914:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828905.702701:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828905.702709:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828905.702785:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828905.702789:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828905.702792:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828906.726730:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828906.726738:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828906.726813:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828906.726817:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828906.726820:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828907.750768:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828907.750793:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828907.750891:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828907.750897:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828907.750902:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787828907.878725:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828907.878736:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1787828907.878755:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1787828907.878760:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787828907.878765:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1787828907.878773:0:468:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 3708 00000100:00080000:1.0:1787828907.878800:0:468:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1787828907.878804:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787828907.878807:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:1.0:1787828907.878812:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:1.0:1787828907.878815:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:1.0:1787828907.878819:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:1.0:1787828907.878822:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:1.0:1787828907.878825:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787828907.878826:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:1.0:1787828907.878833:0:468:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:1.0:1787828907.878836:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828907.878840:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1787828907.878844:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787828907.878865:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828907.878868:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:1.0:1787828907.878871:0:468:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828907.878883:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1787828907.878885:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828907.878889:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1787828907.878892:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1787828907.878897:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828907.878909:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787828907.878920:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d13ecb40 x1874670646776704/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787828907.878934:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787828907.878939:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828907.878957:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1787828907.878960:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828907.878965:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1787828907.878967:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1787828907.878969:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787828907.878973:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1787828907.878976:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3708 00000100:00080000:1.0:1787828907.878982:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1787828907.878986:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787828907.878988:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:1.0:1787828907.878991:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:1.0:1787828907.878993:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:1.0:1787828907.878995:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000020:00000001:1.0:1787828907.878998:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:1.0:1787828907.879001:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787828907.879004:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:2.0:1787828907.879023:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787828907.879029:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1787828907.879032:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00080000:1.0:1787828907.879036:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787828907.879038:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787828907.879043:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828907.879049:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828907.879050:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787828907.879055:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787828907.879057:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828907.879063:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828907.879066:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00100000:2.0:1787828907.879073:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d13ecb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646776704:192.168.203.143@tcp:250:kworker.0 02000000:00000001:1.0:1787828907.879074:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828907.879080:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:1.0:1787828907.879087:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00000001:2.0:1787828907.879087:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787828907.879093:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000008:00000001:1.0:1787828907.879094:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828907.879099:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828907.879100:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828907.879109:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787828907.879111:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1787828907.879115:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828907.879118:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828907.879119:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1787828907.879122:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1787828907.879127:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828907.879134:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787828907.879141:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d13ed680 x1874670646776832/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787828907.879141:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787828907.879147:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839541340992 : -126234168210624 : ffff8d30d13ecb40) 00000100:00000001:1.0:1787828907.879157:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787828907.879158:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:1.0:1787828907.879160:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828907.879166:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1787828907.879169:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828907.879172:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:1.0:1787828907.879173:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1787828907.879175:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787828907.879178:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1787828907.879181:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3708 00000100:00080000:1.0:1787828907.879198:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1787828907.879201:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787828907.879202:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:1.0:1787828907.879204:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:1.0:1787828907.879208:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000800:00000100:2.0:1787828907.879209:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000020:00000001:1.0:1787828907.879211:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000020:00000001:1.0:1787828907.879215:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:1.0:1787828907.879217:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787828907.879219:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:1.0:1787828907.879224:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787828907.879224:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787828907.879227:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828907.879230:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:2.0:1787828907.879231:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 02000000:00000001:1.0:1787828907.879234:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787828907.879236:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828907.879239:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828907.879240:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787828907.879245:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:1.0:1787828907.879249:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00000001:2.0:1787828907.879250:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:1.0:1787828907.879252:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828907.879256:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828907.879259:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4298382080 02000000:00000001:1.0:1787828907.879261:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1787828907.879263:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828907.879267:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1787828907.879268:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1787828907.879269:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828907.879274:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828907.879276:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 00000100:00100000:1.0:1787828907.879280:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d13ed2c0 x1874670646776960/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787828907.879288:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4298382080 00000100:00000001:1.0:1787828907.879289:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787828907.879293:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828907.879297:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1787828907.879299:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:2.0:1787828907.879300:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 00000100:00080000:1.0:1787828907.879305:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1787828907.879309:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787828907.879310:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1787828907.879313:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787828907.879317:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828907.879319:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:2.0:1787828907.879324:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00080000:1.0:1787828907.879328:0:468:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3708 00000100:00100000:2.0:1787828907.879335:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828907/real 1787828907] req@ffff8d30d13ecb40 x1874670646776704/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828932 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787828907.879337:0:468:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1787828907.879346:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787828907.879351:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000400:00000001:2.0:1787828907.879354:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000020:00000001:1.0:1787828907.879356:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000400:00000001:2.0:1787828907.879358:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000020:00000001:1.0:1787828907.879362:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:2.0:1787828907.879364:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000020:00000001:1.0:1787828907.879367:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:2.0:1787828907.879369:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787828907.879372:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:1.0:1787828907.879376:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:2.0:1787828907.879377:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828907.879380:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828907.879384:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00100000:2.0:1787828907.879387:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d13ecb40 x1874670646776704/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828932 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787828907.879389:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:2.0:1787828907.879402:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:1.0:1787828907.879404:0:468:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787828907.879407:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787828907.879412:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828907.879413:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828907.879421:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 02000000:00000001:1.0:1787828907.879422:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:2.0:1787828907.879423:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:1.0:1787828907.879444:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787828907.879445:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787828907.879448:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 02000000:00000001:1.0:1787828907.879453:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787828907.879453:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00000001:1.0:1787828907.879460:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00080000:2.0:1787828907.879463:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:2.0:1787828907.879477:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000008:00100000:1.0:1787828907.879478:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787828907.879480:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d13ecb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646776704:192.168.203.143@tcp:250:kworker.0 00000008:00000001:1.0:1787828907.879487:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828907.879489:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828907.879492:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828907.879493:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828907.879497:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787828907.879499:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787828907.879502:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787828907.879503:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 02000000:00000001:1.0:1787828907.879507:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1787828907.879508:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:2.0:1787828907.879511:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 00000100:00000001:1.0:1787828907.879514:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787828907.879517:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787828907.879519:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787828907.879520:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 02000000:00000001:1.0:1787828907.879524:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828907.879525:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787828907.879528:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828907.879531:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828907.879535:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828907.879538:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1787828907.879558:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828907.879562:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00100000:1.0:1787828907.879564:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d13efc00 x1874670646777088/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787828907.879565:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1787828907.879568:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1787828907.879571:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787828907.879574:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828907.879580:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787828907.879583:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828907.879588:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828907.879589:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00100000:2.0:1787828907.879595:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d13ed680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646776832:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787828907.879597:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828907.879601:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787828907.879605:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:1.0:1787828907.879607:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828907.879608:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828907.879615:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1787828907.879618:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828907.879626:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787828907.879628:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839541343872 : -126234168207744 : ffff8d30d13ed680) 00000100:00000001:2.0:1787828907.879664:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1787828907.879681:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:2.0:1787828907.879686:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1787828907.879689:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828907.879690:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828907.879692:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1787828907.879694:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828907.879698:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298380823 00000100:00000001:2.0:1787828907.879701:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298382080 00000100:00000001:2.0:1787828907.879703:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298380823 00000100:00000001:2.0:1787828907.879705:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298382080 00000100:00000001:2.0:1787828907.879708:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787828907.879709:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828907.879712:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1787828907.879716:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828907/real 1787828907] req@ffff8d30d13ed680 x1874670646776832/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828932 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1787828907.879724:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787828907.879725:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1787828907.879727:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1787828907.879729:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787828907.879730:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828907.879732:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828907.879733:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787828907.879736:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d13ed680 x1874670646776832/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828932 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787828907.879762:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828907.879764:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787828907.879766:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828907.879768:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828907.879769:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828907.879770:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787828907.879772:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787828907.879774:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828907.879789:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:2.0:1787828907.879791:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1787828907.879793:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d13ed680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646776832:192.168.203.143@tcp:38:kworker.0 00000100:00000001:2.0:1787828907.879797:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1787828907.879798:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787828907.879799:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828907.879801:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787828907.879803:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828907.879804:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787828907.879807:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d13ed2c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646776960:192.168.203.143@tcp:38:kworker.0 00000100:00000001:2.0:1787828907.879812:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787828907.879814:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1787828907.879815:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828907.879817:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1787828907.879820:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828907.879825:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787828907.879826:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839541342912 : -126234168208704 : ffff8d30d13ed2c0) 00000100:00000001:2.0:1787828907.879830:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1787828907.879840:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:2.0:1787828907.879845:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1787828907.879847:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828907.879848:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828907.879849:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1787828907.879851:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828907.879854:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298380823 00000100:00000001:2.0:1787828907.879857:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298382080 00000100:00000001:2.0:1787828907.879858:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298380823 00000100:00000001:2.0:1787828907.879860:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298382080 00000100:00000001:2.0:1787828907.879862:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787828907.879864:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828907.879865:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1787828907.879878:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828907/real 1787828907] req@ffff8d30d13ed2c0 x1874670646776960/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828932 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1787828907.879884:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787828907.879885:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1787828907.879887:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1787828907.879889:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787828907.879890:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828907.879892:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828907.879893:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787828907.879896:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d13ed2c0 x1874670646776960/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828932 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787828907.879908:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828907.879909:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787828907.879911:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828907.879913:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828907.879914:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828907.879915:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787828907.879916:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787828907.879918:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828907.879921:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:2.0:1787828907.879922:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1787828907.879925:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d13ed2c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646776960:192.168.203.143@tcp:38:kworker.0 00000100:00000001:2.0:1787828907.879928:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828907.879929:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828907.879930:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787828907.879932:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1787828907.879933:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787828907.879935:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:2.0:1787828907.879936:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:2.0:1787828907.879938:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787828907.879939:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1787828907.879941:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787828907.879943:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828907.879944:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828907.879945:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787828907.879946:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1787828907.879948:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787828907.879949:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:2.0:1787828907.879950:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:2.0:1787828907.879951:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787828907.879952:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1787828907.879953:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787828907.879954:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828907.879955:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828907.879958:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1787828907.879960:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828907.879961:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787828907.879963:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1787828907.879964:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1787828907.879965:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787828907.879966:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828907.879968:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787828907.879969:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828907.879971:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787828907.879973:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d13efc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646777088:192.168.203.143@tcp:8:kworker.0 00000100:00000001:2.0:1787828907.879976:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787828907.879978:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1787828907.879979:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828907.879980:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1787828907.879983:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828907.879987:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787828907.879988:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839541353472 : -126234168198144 : ffff8d30d13efc00) 00000100:00000001:2.0:1787828907.879991:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1787828907.880000:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:2.0:1787828907.880004:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1787828907.880005:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828907.880007:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828907.880008:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1787828907.880010:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828907.880012:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298380836 00000100:00000001:2.0:1787828907.880015:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298382080 00000100:00000001:2.0:1787828907.880016:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298380836 00000100:00000001:2.0:1787828907.880018:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298382080 00000100:00000001:2.0:1787828907.880020:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787828907.880022:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828907.880023:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1787828907.880027:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828907/real 1787828907] req@ffff8d30d13efc00 x1874670646777088/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828932 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1787828907.880039:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787828907.880040:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1787828907.880042:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1787828907.880043:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787828907.880044:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828907.880045:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828907.880046:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787828907.880049:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d13efc00 x1874670646777088/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828932 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787828907.880054:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828907.880056:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787828907.880058:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828907.880059:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828907.880060:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828907.880062:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787828907.880063:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787828907.880066:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828907.880068:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:2.0:1787828907.880070:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1787828907.880072:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d13efc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646777088:192.168.203.143@tcp:8:kworker.0 00000100:00000001:2.0:1787828907.880075:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828907.880076:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828907.880077:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787828907.880078:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1787828907.880080:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787828907.880081:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:2.0:1787828907.880082:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:2.0:1787828907.880083:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787828907.880084:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1787828907.880086:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787828907.880087:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828907.880089:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828907.880091:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1787828907.880092:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828907.880094:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787828907.880095:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828908.774681:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828908.774687:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828908.774765:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828908.774769:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828908.774773:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828909.798699:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828909.798705:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828909.798768:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828909.798772:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828909.798774:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828910.822739:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828910.822745:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828910.822806:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828910.822878:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828910.822881:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828911.846730:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828911.846737:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828911.846820:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828911.846824:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828911.846828:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828912.870695:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828912.870702:0:815:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1787828912.870719:0:815:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1787828912.870723:0:815:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828912.870728:0:815:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1787828912.870730:0:815:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828912.870733:0:815:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828912.870735:0:815:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828912.870792:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828912.870796:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828912.870798:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787828912.998744:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828912.998771:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1787828912.998774:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787828912.998778:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787828912.998782:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787828912.998796:0:40701:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 3713 00000100:00080000:0.0:1787828912.998806:0:40701:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787828912.998810:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787828912.998812:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787828912.998816:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787828912.998819:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787828912.998822:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:0.0:1787828912.998825:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:0.0:1787828912.998827:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787828912.998829:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:0.0:1787828912.998834:0:40701:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787828912.998837:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828912.998840:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787828912.998844:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787828912.998848:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828912.998850:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1787828912.998853:0:40701:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828912.998864:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787828912.998866:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828912.998870:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787828912.998873:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787828912.998877:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828912.998888:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828912.998899:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d26940 x1874670646777216/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787828912.998911:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787828912.998915:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828912.998929:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787828912.998933:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828912.998936:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1787828912.998938:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787828912.998940:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787828912.998944:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787828912.998947:0:40701:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3713 00000100:00080000:0.0:1787828912.998952:0:40701:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787828912.998955:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787828912.998973:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787828912.998976:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787828912.998978:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787828912.998981:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000020:00000001:0.0:1787828912.998985:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:2.0:1787828912.998986:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787828912.998990:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828912.998992:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787828912.998994:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:2.0:1787828912.998996:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1787828912.998999:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787828912.999003:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787828912.999004:0:40701:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 02000000:00000001:2.0:1787828912.999009:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:0.0:1787828912.999010:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828912.999013:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787828912.999016:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787828912.999017:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787828912.999023:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d26940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646777216:192.168.203.143@tcp:250:kworker.0 02000000:00000001:0.0:1787828912.999025:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787828912.999028:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787828912.999032:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828912.999033:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:0.0:1787828912.999038:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 02000000:00000001:2.0:1787828912.999038:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828912.999042:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000008:00100000:0.0:1787828912.999046:0:40701:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 02000000:00000001:2.0:1787828912.999048:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:0.0:1787828912.999053:0:40701:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828912.999057:0:40701:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828912.999063:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:2.0:1787828912.999065:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 02000000:00000001:0.0:1787828912.999067:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828912.999068:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601346880 : -126234108204736 : ffff8d30d4d26940) 00000100:00000001:0.0:1787828912.999072:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:2.0:1787828912.999073:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 02000000:00000001:0.0:1787828912.999077:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787828912.999081:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828912.999089:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828912.999097:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d25680 x1874670646777344/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787828912.999108:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000800:00000100:2.0:1787828912.999112:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:0.0:1787828912.999113:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828912.999137:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787828912.999140:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:2.0:1787828912.999142:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00080000:0.0:1787828912.999147:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1787828912.999147:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828912.999151:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:2.0:1787828912.999151:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828912.999153:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00080000:0.0:1787828912.999155:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787828912.999159:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828912.999160:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:2.0:1787828912.999164:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4298387200 00000100:00080000:0.0:1787828912.999166:0:40701:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3713 00000100:00080000:0.0:1787828912.999172:0:40701:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787828912.999174:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1787828912.999175:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787828912.999179:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:2.0:1787828912.999181:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4298387200 00000020:00000001:0.0:1787828912.999182:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787828912.999186:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:2.0:1787828912.999187:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 00000020:00000001:0.0:1787828912.999189:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:2.0:1787828912.999191:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000020:00000001:0.0:1787828912.999195:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:2.0:1787828912.999195:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828912.999199:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1787828912.999205:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828912/real 1787828912] req@ffff8d30d4d26940 x1874670646777216/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828937 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787828912.999208:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787828912.999213:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000400:00000001:2.0:1787828912.999216:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787828912.999219:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00080000:0.0:1787828912.999221:0:40701:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787828912.999223:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1787828912.999226:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787828912.999228:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828912.999228:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:0.0:1787828912.999234:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:2.0:1787828912.999234:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828912.999238:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:0.0:1787828912.999240:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00100000:2.0:1787828912.999243:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d26940 x1874670646777216/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828937 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:0.0:1787828912.999246:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828912.999251:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:2.0:1787828912.999254:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000008:00100000:0.0:1787828912.999256:0:40701:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00000001:2.0:1787828912.999258:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000008:00000001:0.0:1787828912.999263:0:40701:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828912.999263:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828912.999268:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828912.999269:0:40701:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828912.999270:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828912.999274:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787828912.999276:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 02000000:00000001:0.0:1787828912.999277:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00080000:2.0:1787828912.999278:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 02000000:00000001:0.0:1787828912.999280:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787828912.999283:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787828912.999285:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:2.0:1787828912.999286:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 02000000:00000001:0.0:1787828912.999289:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00100000:2.0:1787828912.999292:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d26940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646777216:192.168.203.143@tcp:250:kworker.0 02000000:00000001:0.0:1787828912.999293:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828912.999297:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828912.999299:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828912.999301:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 00000100:00000001:0.0:1787828912.999302:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828912.999303:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1787828912.999307:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787828912.999309:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:2.0:1787828912.999312:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 00000100:00100000:0.0:1787828912.999314:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d252c0 x1874670646777472/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:2.0:1787828912.999316:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787828912.999319:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1787828912.999323:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787828912.999325:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828912.999328:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828912.999332:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787828912.999335:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828912.999336:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1787828912.999339:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828912.999340:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828912.999341:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787828912.999344:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00080000:0.0:1787828912.999347:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:2.0:1787828912.999347:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1787828912.999349:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787828912.999351:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787828912.999356:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 02000000:00000001:2.0:1787828912.999356:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787828912.999359:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787828912.999361:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 02000000:00000001:2.0:1787828912.999362:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787828912.999365:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1787828912.999367:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d25680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646777344:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787828912.999370:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:2.0:1787828912.999371:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787828912.999373:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1787828912.999375:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787828912.999376:0:40701:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3713 02000000:00000001:2.0:1787828912.999377:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1787828912.999380:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787828912.999383:0:40701:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787828912.999386:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787828912.999387:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828912.999388:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601342080 : -126234108209536 : ffff8d30d4d25680) 00000100:00000001:0.0:1787828912.999391:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:2.0:1787828912.999393:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000020:00000001:0.0:1787828912.999397:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787828912.999400:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787828912.999403:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000800:00000100:2.0:1787828912.999407:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000020:00000001:0.0:1787828912.999409:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:2.0:1787828912.999412:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1787828912.999413:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828912.999415:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828912.999417:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:2.0:1787828912.999418:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828912.999422:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1787828912.999425:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787828912.999426:0:40701:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787828912.999430:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298385942 00000100:00000001:0.0:1787828912.999431:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828912.999434:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298387200 02000000:00000001:0.0:1787828912.999435:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:2.0:1787828912.999437:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298385942 02000000:00000001:0.0:1787828912.999440:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787828912.999441:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298387200 00000100:00000001:2.0:1787828912.999445:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787828912.999447:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828912.999448:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828912.999453:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:2.0:1787828912.999453:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000008:00100000:0.0:1787828912.999458:0:40701:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787828912.999459:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828912/real 1787828912] req@ffff8d30d4d25680 x1874670646777344/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828937 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00000001:0.0:1787828912.999466:0:40701:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:2.0:1787828912.999472:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000001:0.0:1787828912.999473:0:40701:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:2.0:1787828912.999475:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1787828912.999478:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1787828912.999479:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 02000000:00000001:0.0:1787828912.999481:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:2.0:1787828912.999481:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:0.0:1787828912.999486:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828912.999486:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828912.999487:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828912.999492:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00100000:2.0:1787828912.999493:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d25680 x1874670646777344/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828937 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:0.0:1787828912.999495:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787828912.999499:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828912.999502:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828912.999504:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787828912.999506:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828912.999507:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828912.999508:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828912.999510:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828912.999513:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787828912.999515:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787828912.999519:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:0.0:1787828912.999520:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d27c00 x1874670646777600/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787828912.999523:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:2.0:1787828912.999527:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1787828912.999533:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d25680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646777344:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787828912.999542:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787828912.999543:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1787828912.999544:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787828912.999546:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828912.999549:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828912.999549:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787828912.999552:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828912.999555:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828912.999558:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787828912.999558:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d252c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646777472:192.168.203.143@tcp:38:kworker.0 00000100:00000001:2.0:1787828912.999565:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787828912.999566:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1787828912.999567:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828912.999569:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1787828912.999572:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828912.999575:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787828912.999576:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601341120 : -126234108210496 : ffff8d30d4d252c0) 00000100:00000001:2.0:1787828912.999578:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1787828912.999586:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:2.0:1787828912.999590:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1787828912.999593:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828912.999594:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828912.999595:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1787828912.999597:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828912.999599:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298385942 00000100:00000001:2.0:1787828912.999600:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298387200 00000100:00000001:2.0:1787828912.999602:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298385942 00000100:00000001:2.0:1787828912.999604:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298387200 00000100:00000001:2.0:1787828912.999605:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787828912.999606:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828912.999607:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1787828912.999610:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828912/real 1787828912] req@ffff8d30d4d252c0 x1874670646777472/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828937 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1787828912.999644:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787828912.999645:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1787828912.999647:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1787828912.999648:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787828912.999649:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828912.999650:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828912.999652:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787828912.999654:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d252c0 x1874670646777472/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828937 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787828912.999659:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828912.999660:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787828912.999662:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828912.999663:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828912.999664:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828912.999665:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787828912.999666:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787828912.999667:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828912.999669:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:2.0:1787828912.999670:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1787828912.999672:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d252c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646777472:192.168.203.143@tcp:38:kworker.0 00000100:00000001:2.0:1787828912.999674:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828912.999675:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828912.999676:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787828912.999677:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1787828912.999679:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787828912.999680:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:2.0:1787828912.999681:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:2.0:1787828912.999682:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787828912.999683:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1787828912.999685:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787828912.999692:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828912.999694:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828912.999695:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787828912.999696:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1787828912.999697:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787828912.999698:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:2.0:1787828912.999699:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:2.0:1787828912.999700:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787828912.999701:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1787828912.999701:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787828912.999718:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828912.999720:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828912.999723:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1787828912.999724:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828912.999726:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787828912.999727:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1787828912.999729:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1787828912.999730:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787828912.999732:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828912.999734:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787828912.999736:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828912.999739:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787828912.999742:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d27c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646777600:192.168.203.143@tcp:8:kworker.0 00000100:00000001:2.0:1787828912.999746:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787828912.999748:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1787828912.999749:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828912.999751:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1787828912.999754:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828912.999760:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787828912.999762:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601351680 : -126234108199936 : ffff8d30d4d27c00) 00000100:00000001:2.0:1787828912.999765:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1787828912.999777:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:2.0:1787828912.999780:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1787828912.999782:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828912.999783:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828912.999791:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1787828912.999793:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828912.999795:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298385956 00000100:00000001:2.0:1787828912.999797:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298387200 00000100:00000001:2.0:1787828912.999799:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298385956 00000100:00000001:2.0:1787828912.999800:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298387200 00000100:00000001:2.0:1787828912.999802:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787828912.999803:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828912.999804:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1787828912.999807:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828912/real 1787828912] req@ffff8d30d4d27c00 x1874670646777600/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828937 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1787828912.999812:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787828912.999813:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1787828912.999815:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1787828912.999816:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787828912.999817:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828912.999818:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828912.999819:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787828912.999821:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d27c00 x1874670646777600/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828937 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787828912.999826:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828912.999827:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787828912.999828:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828912.999829:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828912.999830:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828912.999831:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787828912.999833:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787828912.999834:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828912.999837:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:2.0:1787828912.999839:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1787828912.999842:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d27c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646777600:192.168.203.143@tcp:8:kworker.0 00000100:00000001:2.0:1787828912.999845:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828912.999847:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828912.999848:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787828912.999850:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1787828912.999852:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787828912.999854:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:2.0:1787828912.999855:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:2.0:1787828912.999857:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787828912.999858:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1787828912.999861:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787828912.999863:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828912.999865:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828912.999868:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1787828912.999869:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828912.999871:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787828912.999873:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828913.894709:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828913.894716:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828913.894766:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828913.894770:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828913.894774:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828914.918759:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828914.918767:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828914.918821:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828914.918825:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828914.918829:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828915.942742:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828915.942749:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828915.942799:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828915.942802:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828915.942805:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828916.966703:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828916.966711:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828916.966760:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828916.966764:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828916.966768:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828917.990760:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828917.990768:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828917.990839:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828917.990844:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828917.990849:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787828918.118720:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828918.118730:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1787828918.118732:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787828918.118737:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787828918.118741:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787828918.118749:0:40701:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 3718 00000100:00080000:0.0:1787828918.118760:0:40701:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787828918.118763:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787828918.118765:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787828918.118769:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787828918.118771:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787828918.118774:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:0.0:1787828918.118776:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:0.0:1787828918.118778:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787828918.118779:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:0.0:1787828918.118784:0:40701:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787828918.118786:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828918.118788:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787828918.118792:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787828918.118795:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828918.118797:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1787828918.118799:0:40701:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828918.118813:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787828918.118814:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828918.118817:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787828918.118820:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787828918.118825:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828918.118837:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828918.118847:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d270c0 x1874670646777728/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787828918.118856:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787828918.118860:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828918.118878:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787828918.118880:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828918.118883:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1787828918.118884:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787828918.118886:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787828918.118888:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787828918.118889:0:40701:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3718 00000100:00080000:0.0:1787828918.118891:0:40701:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787828918.118893:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787828918.118894:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787828918.118895:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787828918.118896:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787828918.118898:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000020:00000001:0.0:1787828918.118900:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:0.0:1787828918.118901:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787828918.118903:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:0.0:1787828918.118905:0:40701:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787828918.118907:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828918.118908:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787828918.118914:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787828918.118915:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828918.118917:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1787828918.118921:0:40701:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1787828918.118923:0:40701:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828918.118925:0:40701:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828918.118928:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:2.0:1787828918.118928:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:0.0:1787828918.118930:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828918.118933:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787828918.118934:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:2.0:1787828918.118935:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 02000000:00000001:0.0:1787828918.118936:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828918.118939:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1787828918.118941:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828918.118942:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787828918.118946:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828918.118947:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d24000 x1874670646777856/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:2.0:1787828918.118951:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787828918.118955:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787828918.118958:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 02000000:00000001:2.0:1787828918.118959:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828918.118962:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828918.118966:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787828918.118967:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d270c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646777728:192.168.203.143@tcp:250:kworker.0 00000100:00080000:0.0:1787828918.118968:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828918.118975:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1787828918.118977:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:2.0:1787828918.118977:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00080000:0.0:1787828918.118981:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787828918.118983:0:40701:0:(import.c:485:import_select_connection()) Process entered 02000000:00000001:2.0:1787828918.118984:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000100:00080000:0.0:1787828918.118986:0:40701:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3718 02000000:00000001:2.0:1787828918.118988:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787828918.118990:0:40701:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 02000000:00000001:2.0:1787828918.118991:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:0.0:1787828918.118992:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787828918.118995:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787828918.118996:0:40701:0:(genops.c:792:class_conn2export()) Process entered 02000000:00000001:2.0:1787828918.118998:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787828918.118999:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787828918.119002:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000020:00000001:0.0:1787828918.119004:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:0.0:1787828918.119006:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787828918.119007:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:0.0:1787828918.119009:0:40701:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787828918.119011:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828918.119012:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787828918.119014:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787828918.119015:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828918.119016:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1787828918.119018:0:40701:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00000001:2.0:1787828918.119019:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000008:00000001:0.0:1787828918.119020:0:40701:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828918.119023:0:40701:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828918.119024:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601348800 : -126234108202816 : ffff8d30d4d270c0) 02000000:00000001:0.0:1787828918.119028:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787828918.119030:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828918.119030:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:0.0:1787828918.119034:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787828918.119034:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787828918.119036:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828918.119041:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828918.119044:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d24f00 x1874670646777984/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787828918.119050:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787828918.119051:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828918.119053:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787828918.119055:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828918.119057:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1787828918.119058:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787828918.119059:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787828918.119060:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787828918.119062:0:40701:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3718 00000100:00080000:0.0:1787828918.119067:0:40701:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787828918.119069:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787828918.119070:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787828918.119072:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787828918.119073:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787828918.119074:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000020:00000001:0.0:1787828918.119076:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:0.0:1787828918.119078:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787828918.119080:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000800:00000100:2.0:1787828918.119081:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00080000:0.0:1787828918.119085:0:40701:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787828918.119088:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828918.119089:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787828918.119091:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787828918.119092:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828918.119094:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1787828918.119095:0:40701:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1787828918.119097:0:40701:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828918.119098:0:40701:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828918.119100:0:2390:0:(events.c:34:request_out_callback()) Process entered 02000000:00000001:0.0:1787828918.119102:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787828918.119104:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828918.119105:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828918.119106:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787828918.119109:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:2.0:1787828918.119110:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828918.119112:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828918.119113:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1787828918.119120:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828918.119124:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828918.119125:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4298392320 00000100:00100000:0.0:1787828918.119130:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d25a40 x1874670646778112/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787828918.119136:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787828918.119138:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1787828918.119139:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828918.119144:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828918.119150:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4298392320 00000100:00000001:2.0:1787828918.119155:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 00000100:00000001:2.0:1787828918.119159:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787828918.119162:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828918.119164:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1787828918.119169:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828918/real 1787828918] req@ffff8d30d4d270c0 x1874670646777728/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828943 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1787828918.119180:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787828918.119182:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1787828918.119186:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1787828918.119189:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787828918.119191:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828918.119195:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828918.119197:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787828918.119202:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d270c0 x1874670646777728/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828943 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787828918.119211:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828918.119215:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787828918.119219:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828918.119220:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828918.119222:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828918.119225:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787828918.119227:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787828918.119231:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828918.119236:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:2.0:1787828918.119238:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1787828918.119242:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d270c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646777728:192.168.203.143@tcp:250:kworker.0 00000100:00000001:2.0:1787828918.119246:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828918.119248:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828918.119249:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787828918.119252:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1787828918.119255:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787828918.119257:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:2.0:1787828918.119259:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:2.0:1787828918.119263:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787828918.119265:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1787828918.119269:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787828918.119270:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828918.119272:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828918.119280:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1787828918.119282:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828918.119283:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787828918.119284:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1787828918.119285:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1787828918.119287:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787828918.119288:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828918.119289:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787828918.119291:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828918.119293:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787828918.119294:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d24000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646777856:192.168.203.143@tcp:38:kworker.0 00000100:00000001:2.0:1787828918.119297:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787828918.119299:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1787828918.119300:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828918.119301:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1787828918.119303:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828918.119308:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787828918.119309:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601336320 : -126234108215296 : ffff8d30d4d24000) 00000100:00000001:2.0:1787828918.119314:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1787828918.119324:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:2.0:1787828918.119327:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1787828918.119329:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828918.119330:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828918.119331:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1787828918.119333:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828918.119337:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298391062 00000100:00000001:2.0:1787828918.119339:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298392320 00000100:00000001:2.0:1787828918.119341:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298391062 00000100:00000001:2.0:1787828918.119343:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298392320 00000100:00000001:2.0:1787828918.119344:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787828918.119346:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828918.119347:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1787828918.119349:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828918/real 1787828918] req@ffff8d30d4d24000 x1874670646777856/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828943 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1787828918.119354:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787828918.119355:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1787828918.119357:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1787828918.119358:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787828918.119359:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828918.119360:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828918.119361:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787828918.119364:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d24000 x1874670646777856/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828943 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787828918.119368:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828918.119370:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787828918.119371:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828918.119372:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828918.119373:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828918.119374:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787828918.119375:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787828918.119377:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828918.119380:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:2.0:1787828918.119381:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1787828918.119383:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d24000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646777856:192.168.203.143@tcp:38:kworker.0 00000100:00000001:2.0:1787828918.119386:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1787828918.119387:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787828918.119388:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828918.119389:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787828918.119390:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828918.119391:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787828918.119393:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d24f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646777984:192.168.203.143@tcp:38:kworker.0 00000100:00000001:2.0:1787828918.119395:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787828918.119396:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1787828918.119396:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828918.119398:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1787828918.119400:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828918.119402:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787828918.119403:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601340160 : -126234108211456 : ffff8d30d4d24f00) 00000100:00000001:2.0:1787828918.119406:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1787828918.119411:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:2.0:1787828918.119414:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1787828918.119415:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828918.119416:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828918.119418:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1787828918.119419:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828918.119421:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298391062 00000100:00000001:2.0:1787828918.119423:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298392320 00000100:00000001:2.0:1787828918.119425:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298391062 00000100:00000001:2.0:1787828918.119426:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298392320 00000100:00000001:2.0:1787828918.119428:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787828918.119429:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828918.119430:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1787828918.119433:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828918/real 1787828918] req@ffff8d30d4d24f00 x1874670646777984/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828943 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1787828918.119438:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787828918.119439:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1787828918.119441:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1787828918.119442:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787828918.119442:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828918.119444:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828918.119445:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787828918.119447:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d24f00 x1874670646777984/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828943 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787828918.119452:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828918.119453:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787828918.119455:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828918.119456:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828918.119456:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828918.119457:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787828918.119458:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787828918.119460:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828918.119461:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:2.0:1787828918.119463:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1787828918.119464:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d24f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646777984:192.168.203.143@tcp:38:kworker.0 00000100:00000001:2.0:1787828918.119466:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1787828918.119467:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787828918.119468:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828918.119469:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787828918.119470:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828918.119471:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787828918.119473:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d25a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646778112:192.168.203.143@tcp:8:kworker.0 00000100:00000001:2.0:1787828918.119475:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787828918.119476:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1787828918.119477:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828918.119478:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1787828918.119480:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828918.119484:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787828918.119485:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601343040 : -126234108208576 : ffff8d30d4d25a40) 00000100:00000001:2.0:1787828918.119487:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1787828918.119492:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:2.0:1787828918.119494:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1787828918.119496:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828918.119497:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828918.119498:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1787828918.119499:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828918.119501:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298391075 00000100:00000001:2.0:1787828918.119503:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298392320 00000100:00000001:2.0:1787828918.119505:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298391075 00000100:00000001:2.0:1787828918.119506:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298392320 00000100:00000001:2.0:1787828918.119507:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787828918.119509:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828918.119510:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1787828918.119512:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828918/real 1787828918] req@ffff8d30d4d25a40 x1874670646778112/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828943 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1787828918.119516:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787828918.119517:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1787828918.119518:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1787828918.119519:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787828918.119520:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828918.119521:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828918.119522:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787828918.119524:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d25a40 x1874670646778112/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828943 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787828918.119528:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828918.119529:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787828918.119531:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828918.119532:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828918.119533:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828918.119535:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787828918.119536:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787828918.119538:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828918.119539:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:2.0:1787828918.119540:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1787828918.119542:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d25a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646778112:192.168.203.143@tcp:8:kworker.0 00000100:00000001:2.0:1787828918.119544:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828918.119545:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828918.119546:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787828918.119547:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1787828918.119548:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787828918.119549:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:2.0:1787828918.119550:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:2.0:1787828918.119551:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787828918.119552:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1787828918.119554:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787828918.119554:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828918.119555:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828918.119556:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787828918.119557:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1787828918.119558:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787828918.119559:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:2.0:1787828918.119559:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:2.0:1787828918.119560:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787828918.119561:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1787828918.119563:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787828918.119563:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828918.119565:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828918.119565:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787828918.119566:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1787828918.119567:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787828918.119568:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:2.0:1787828918.119569:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:2.0:1787828918.119570:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787828918.119571:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1787828918.119572:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787828918.119573:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828918.119574:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828918.119576:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1787828918.119577:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828918.119579:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787828918.119580:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828919.014696:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828919.014713:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828919.014770:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828919.014774:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828919.014777:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828920.038756:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828920.038781:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828920.038871:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828920.038876:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828920.038880:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828921.062706:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828921.062712:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828921.062764:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828921.062780:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828921.062784:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828922.086698:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828922.086703:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828922.086755:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828922.086759:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828922.086761:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828923.110720:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828923.110726:0:815:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1787828923.110728:0:815:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1787828923.110730:0:815:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828923.110734:0:815:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1787828923.110736:0:815:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828923.110737:0:815:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828923.110739:0:815:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828923.110790:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828923.110794:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828923.110796:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787828923.238792:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828923.238802:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1787828923.238805:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787828923.238809:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787828923.238822:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787828923.238830:0:40701:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 3723 00000100:00080000:0.0:1787828923.238842:0:40701:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787828923.238845:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787828923.238848:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787828923.238852:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787828923.238854:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787828923.238858:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:0.0:1787828923.238859:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:0.0:1787828923.238861:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787828923.238862:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:0.0:1787828923.238866:0:40701:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787828923.238869:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828923.238871:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787828923.238874:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787828923.238877:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828923.238879:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1787828923.238881:0:40701:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828923.238912:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787828923.238914:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828923.238918:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787828923.238921:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787828923.238926:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828923.238942:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828923.238968:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d25a40 x1874670646778240/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787828923.238980:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787828923.238986:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828923.239006:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787828923.239009:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828923.239019:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1787828923.239021:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787828923.239023:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787828923.239030:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787828923.239033:0:40701:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3723 00000100:00080000:0.0:1787828923.239038:0:40701:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787828923.239041:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787828923.239060:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787828923.239062:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787828923.239064:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787828923.239066:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000020:00000001:0.0:1787828923.239068:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:0.0:1787828923.239070:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787828923.239071:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:0.0:1787828923.239074:0:40701:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787828923.239076:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828923.239077:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:2.0:1787828923.239080:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:0.0:1787828923.239085:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787828923.239089:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 02000000:00000001:0.0:1787828923.239092:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828923.239093:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1787828923.239098:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:2.0:1787828923.239098:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787828923.239104:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828923.239122:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000008:00100000:0.0:1787828923.239124:0:40701:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 02000000:00000001:2.0:1787828923.239126:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828923.239133:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:0.0:1787828923.239136:0:40701:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828923.239143:0:40701:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787828923.239143:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d25a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646778240:192.168.203.143@tcp:250:kworker.0 00000100:00000001:2.0:1787828923.239156:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787828923.239159:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787828923.239161:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787828923.239166:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828923.239167:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828923.239175:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:0.0:1787828923.239192:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787828923.239195:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787828923.239195:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828923.239201:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828923.239213:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828923.239216:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787828923.239220:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601343040 : -126234108208576 : ffff8d30d4d25a40) 00000100:00100000:0.0:1787828923.239225:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d24f00 x1874670646778368/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787828923.239227:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:0.0:1787828923.239240:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787828923.239244:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828923.239249:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787828923.239253:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828923.239256:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1787828923.239258:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787828923.239260:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787828923.239263:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787828923.239266:0:40701:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3723 00000100:00080000:0.0:1787828923.239270:0:40701:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787828923.239272:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787828923.239274:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787828923.239278:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787828923.239281:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000800:00000100:2.0:1787828923.239282:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000020:00000001:0.0:1787828923.239284:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000020:00000001:0.0:1787828923.239290:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:0.0:1787828923.239293:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787828923.239295:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:0.0:1787828923.239299:0:40701:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787828923.239302:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828923.239304:0:2390:0:(events.c:34:request_out_callback()) Process entered 02000000:00000001:0.0:1787828923.239305:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787828923.239327:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787828923.239332:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828923.239333:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828923.239334:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:2.0:1787828923.239338:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000008:00100000:0.0:1787828923.239339:0:40701:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00000001:2.0:1787828923.239342:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000008:00000001:0.0:1787828923.239347:0:40701:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828923.239352:0:40701:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828923.239354:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828923.239357:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787828923.239360:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828923.239361:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4298397440 00000100:00000001:0.0:1787828923.239363:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787828923.239370:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787828923.239374:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828923.239377:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1787828923.239381:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828923.239386:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4298397440 00000100:00100000:0.0:1787828923.239388:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d24000 x1874670646778496/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787828923.239392:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 00000100:00000001:2.0:1787828923.239400:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787828923.239412:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828923.239420:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00000001:0.0:1787828923.239431:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00100000:2.0:1787828923.239433:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828923/real 1787828923] req@ffff8d30d4d25a40 x1874670646778240/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828948 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787828923.239437:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828923.239445:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787828923.239455:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787828923.239469:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000400:00000001:2.0:1787828923.239470:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000001:0.0:1787828923.239471:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000400:00000001:2.0:1787828923.239474:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00080000:0.0:1787828923.239475:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787828923.239482:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:2.0:1787828923.239483:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1787828923.239494:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00080000:0.0:1787828923.239496:0:40701:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3723 00000100:00000001:2.0:1787828923.239510:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828923.239513:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828923.239520:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:0.0:1787828923.239522:0:40701:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787828923.239525:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787828923.239530:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00100000:2.0:1787828923.239530:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d25a40 x1874670646778240/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828948 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:0.0:1787828923.239541:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787828923.239546:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787828923.239552:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:2.0:1787828923.239557:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1787828923.239558:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:2.0:1787828923.239563:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787828923.239578:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828923.239581:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828923.239581:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828923.239593:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:2.0:1787828923.239595:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:0.0:1787828923.239606:0:40701:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787828923.239608:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828923.239610:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787828923.239614:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787828923.239616:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828923.239617:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1787828923.239619:0:40701:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1787828923.239621:0:40701:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828923.239623:0:40701:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828923.239627:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787828923.239629:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828923.239665:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787828923.239668:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787828923.239671:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828923.239676:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:2.0:1787828923.239678:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1787828923.239681:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d25a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646778240:192.168.203.143@tcp:250:kworker.0 00000100:00000001:2.0:1787828923.239688:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828923.239690:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828923.239692:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:2.0:1787828923.239692:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787828923.239699:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787828923.239700:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787828923.239710:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828923.239710:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787828923.239716:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:2.0:1787828923.239730:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:2.0:1787828923.239734:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787828923.239736:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787828923.239738:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828923.239740:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787828923.239742:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828923.239745:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787828923.239749:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d270c0 x1874670646778624/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787828923.239754:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1787828923.239760:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828923.239773:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787828923.239774:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787828923.239781:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828923.239781:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1787828923.239789:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1787828923.239792:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828923.239792:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787828923.239799:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828923.239803:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787828923.239806:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828923.239809:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787828923.239813:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d24f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646778368:192.168.203.143@tcp:38:kworker.0 00000100:00000001:2.0:1787828923.239817:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787828923.239819:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1787828923.239821:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828923.239822:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1787828923.239825:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828923.239830:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787828923.239831:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601340160 : -126234108211456 : ffff8d30d4d24f00) 00000100:00000001:2.0:1787828923.239835:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1787828923.239848:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:2.0:1787828923.239851:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1787828923.239854:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828923.239855:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828923.239856:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1787828923.239858:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828923.239861:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298396183 00000100:00000001:2.0:1787828923.239863:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298397440 00000100:00000001:2.0:1787828923.239865:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298396183 00000100:00000001:2.0:1787828923.239866:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298397440 00000100:00000001:2.0:1787828923.239868:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787828923.239869:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828923.239870:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1787828923.239873:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828923/real 1787828923] req@ffff8d30d4d24f00 x1874670646778368/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828948 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1787828923.239879:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787828923.239880:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1787828923.239882:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1787828923.239883:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787828923.239896:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828923.239897:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828923.239898:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787828923.239900:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d24f00 x1874670646778368/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828948 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787828923.239905:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828923.239906:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787828923.239908:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828923.239909:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828923.239909:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828923.239910:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787828923.239912:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787828923.239914:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828923.239916:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:2.0:1787828923.239918:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1787828923.239920:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d24f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646778368:192.168.203.143@tcp:38:kworker.0 00000100:00000001:2.0:1787828923.239922:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1787828923.239923:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787828923.239924:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828923.239926:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787828923.239927:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828923.239928:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787828923.239930:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d24000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646778496:192.168.203.143@tcp:38:kworker.0 00000100:00000001:2.0:1787828923.239932:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787828923.239933:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1787828923.239934:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828923.239935:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1787828923.239937:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828923.239940:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787828923.239941:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601336320 : -126234108215296 : ffff8d30d4d24000) 00000100:00000001:2.0:1787828923.239943:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1787828923.239949:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:2.0:1787828923.239952:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1787828923.239953:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828923.239954:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828923.239955:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1787828923.239957:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828923.239959:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298396183 00000100:00000001:2.0:1787828923.239961:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298397440 00000100:00000001:2.0:1787828923.239962:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298396183 00000100:00000001:2.0:1787828923.239964:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298397440 00000100:00000001:2.0:1787828923.239977:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787828923.239978:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828923.239979:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1787828923.239982:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828923/real 1787828923] req@ffff8d30d4d24000 x1874670646778496/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828948 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1787828923.239988:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787828923.239988:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1787828923.239990:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1787828923.239991:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787828923.239992:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828923.239993:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828923.239994:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787828923.239996:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d24000 x1874670646778496/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828948 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787828923.240000:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828923.240002:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787828923.240003:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828923.240004:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828923.240005:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828923.240006:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787828923.240007:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787828923.240008:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828923.240010:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:2.0:1787828923.240011:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1787828923.240013:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d24000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646778496:192.168.203.143@tcp:38:kworker.0 00000100:00000001:2.0:1787828923.240015:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828923.240017:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828923.240017:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787828923.240018:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1787828923.240020:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787828923.240020:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:2.0:1787828923.240021:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:2.0:1787828923.240023:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787828923.240023:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1787828923.240026:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787828923.240027:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828923.240028:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828923.240029:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787828923.240030:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1787828923.240032:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787828923.240033:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:2.0:1787828923.240034:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:2.0:1787828923.240036:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787828923.240037:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1787828923.240039:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787828923.240040:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828923.240043:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828923.240046:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1787828923.240047:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828923.240050:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787828923.240051:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1787828923.240053:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1787828923.240054:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787828923.240056:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828923.240058:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787828923.240071:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828923.240073:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787828923.240077:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d270c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646778624:192.168.203.143@tcp:8:kworker.0 00000100:00000001:2.0:1787828923.240081:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787828923.240084:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1787828923.240085:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828923.240087:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1787828923.240090:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828923.240096:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787828923.240097:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601348800 : -126234108202816 : ffff8d30d4d270c0) 00000100:00000001:2.0:1787828923.240100:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1787828923.240109:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:2.0:1787828923.240113:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1787828923.240114:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828923.240115:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828923.240116:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1787828923.240118:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828923.240120:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298396196 00000100:00000001:2.0:1787828923.240123:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298397440 00000100:00000001:2.0:1787828923.240124:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298396196 00000100:00000001:2.0:1787828923.240136:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298397440 00000100:00000001:2.0:1787828923.240137:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787828923.240139:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828923.240140:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1787828923.240143:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828923/real 1787828923] req@ffff8d30d4d270c0 x1874670646778624/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828948 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1787828923.240147:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787828923.240148:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1787828923.240150:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1787828923.240151:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787828923.240152:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828923.240153:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828923.240154:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787828923.240156:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d270c0 x1874670646778624/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828948 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787828923.240161:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828923.240162:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787828923.240164:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828923.240165:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828923.240166:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828923.240167:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787828923.240168:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787828923.240169:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828923.240171:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:2.0:1787828923.240172:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1787828923.240174:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d270c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646778624:192.168.203.143@tcp:8:kworker.0 00000100:00000001:2.0:1787828923.240176:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828923.240178:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828923.240179:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787828923.240180:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1787828923.240181:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787828923.240182:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:2.0:1787828923.240183:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:2.0:1787828923.240184:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787828923.240185:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1787828923.240188:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787828923.240188:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828923.240190:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828923.240192:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1787828923.240193:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828923.240194:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787828923.240195:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828924.134709:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828924.134714:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828924.134768:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828924.134772:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828924.134774:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828925.158672:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828925.158677:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828925.158732:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828925.158736:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828925.158739:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828926.182729:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828926.182735:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828926.182804:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828926.182811:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828926.182816:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828927.206769:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828927.206776:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828927.206844:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828927.206848:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828927.206861:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828928.230712:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828928.230723:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828928.230787:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828928.230791:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828928.230793:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787828928.358759:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828928.358770:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1787828928.358772:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787828928.358776:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787828928.358782:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787828928.358789:0:40701:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 3728 00000100:00080000:2.0:1787828928.358800:0:40701:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787828928.358803:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828928.358805:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787828928.358809:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787828928.358811:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787828928.358814:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:2.0:1787828928.358817:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:2.0:1787828928.358818:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828928.358819:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:2.0:1787828928.358823:0:40701:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787828928.358825:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828928.358827:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787828928.358830:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828928.358833:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828928.358834:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:2.0:1787828928.358837:0:40701:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828928.358850:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787828928.358851:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828928.358854:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787828928.358857:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787828928.358861:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828928.358874:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787828928.358883:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2fc00 x1874670646778752/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787828928.358892:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787828928.358896:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828928.358913:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787828928.358916:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828928.358918:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1787828928.358920:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787828928.358921:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787828928.358923:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787828928.358925:0:40701:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3728 00000100:00080000:2.0:1787828928.358927:0:40701:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787828928.358928:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828928.358930:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787828928.358931:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787828928.358934:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787828928.358936:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000020:00000001:2.0:1787828928.358938:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:2.0:1787828928.358939:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828928.358941:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:2.0:1787828928.358943:0:40701:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787828928.358944:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828928.358945:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:0.0:1787828928.358949:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:2.0:1787828928.358951:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828928.358955:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828928.358956:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1787828928.358957:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000008:00100000:2.0:1787828928.358960:0:40701:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00000001:0.0:1787828928.358962:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000008:00000001:2.0:1787828928.358964:0:40701:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828928.358966:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787828928.358968:0:40701:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828928.358969:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828928.358972:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787828928.358975:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787828928.358975:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828928.358977:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787828928.358979:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828928.358980:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787828928.358984:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828928.358985:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828928.358990:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828928.358994:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646778752:192.168.203.143@tcp:250:kworker.0 00000100:00100000:2.0:1787828928.358996:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2c780 x1874670646778880/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787828928.359005:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:2.0:1787828928.359007:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787828928.359010:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828928.359013:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:2.0:1787828928.359013:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787828928.359015:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 02000000:00000001:0.0:1787828928.359019:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787828928.359019:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1787828928.359022:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787828928.359025:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 02000000:00000001:0.0:1787828928.359027:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:2.0:1787828928.359028:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787828928.359032:0:40701:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3728 00000100:00080000:2.0:1787828928.359034:0:40701:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787828928.359036:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 02000000:00000001:0.0:1787828928.359037:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828928.359037:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787828928.359043:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787828928.359044:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787828928.359045:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000020:00000001:2.0:1787828928.359047:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:2.0:1787828928.359048:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828928.359050:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:2.0:1787828928.359052:0:40701:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787828928.359053:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828928.359054:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787828928.359056:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828928.359057:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828928.359059:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787828928.359059:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1787828928.359060:0:40701:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00000001:0.0:1787828928.359062:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417883648 : -126234291667968 : ffff8d30c9e2fc00) 00000008:00000001:2.0:1787828928.359064:0:40701:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828928.359067:0:40701:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828928.359069:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 02000000:00000001:2.0:1787828928.359071:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787828928.359073:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828928.359075:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787828928.359075:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787828928.359077:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828928.359081:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787828928.359084:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2d2c0 x1874670646779008/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787828928.359089:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787828928.359091:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828928.359092:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787828928.359094:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828928.359096:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1787828928.359097:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787828928.359098:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787828928.359101:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787828928.359103:0:40701:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3728 00000100:00080000:2.0:1787828928.359106:0:40701:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787828928.359107:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828928.359108:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787828928.359109:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787828928.359110:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787828928.359112:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000020:00000001:2.0:1787828928.359115:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000800:00000100:0.0:1787828928.359116:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:2.0:1787828928.359118:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828928.359121:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:2.0:1787828928.359124:0:40701:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787828928.359126:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828928.359128:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787828928.359130:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828928.359131:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828928.359132:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1787828928.359133:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1787828928.359137:0:40701:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000008:00000001:2.0:1787828928.359138:0:40701:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828928.359139:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828928.359140:0:40701:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828928.359143:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828928.359145:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:0.0:1787828928.359147:0:2390:0:(events.c:68:request_out_callback()) Process leaving 02000000:00000001:2.0:1787828928.359148:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828928.359150:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787828928.359151:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787828928.359153:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828928.359154:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828928.359157:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828928.359159:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4298402560 00000100:00100000:2.0:1787828928.359162:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2cb40 x1874670646779136/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787828928.359168:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787828928.359170:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828928.359172:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 00000100:00000001:2.0:1787828928.359172:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828928.359182:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4298402560 00000100:00000001:0.0:1787828928.359189:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1787828928.359193:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787828928.359195:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828928.359198:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787828928.359204:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828928/real 1787828928] req@ffff8d30c9e2fc00 x1874670646778752/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828953 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787828928.359214:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787828928.359215:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787828928.359217:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787828928.359219:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787828928.359221:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828928.359223:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828928.359225:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787828928.359228:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2fc00 x1874670646778752/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828953 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787828928.359233:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828928.359236:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787828928.359238:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828928.359239:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828928.359240:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828928.359242:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787828928.359243:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787828928.359246:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828928.359249:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787828928.359250:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787828928.359253:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646778752:192.168.203.143@tcp:250:kworker.0 00000100:00000001:0.0:1787828928.359257:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828928.359259:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828928.359260:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787828928.359262:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787828928.359265:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787828928.359267:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787828928.359269:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787828928.359274:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787828928.359275:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787828928.359278:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787828928.359280:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828928.359281:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828928.359289:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787828928.359290:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828928.359292:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787828928.359294:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787828928.359295:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1787828928.359296:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787828928.359297:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828928.359298:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787828928.359300:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787828928.359302:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828928.359304:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646778880:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787828928.359306:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787828928.359307:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787828928.359309:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828928.359310:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1787828928.359312:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828928.359316:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787828928.359317:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417870208 : -126234291681408 : ffff8d30c9e2c780) 00000100:00000001:0.0:1787828928.359321:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1787828928.359329:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:0.0:1787828928.359332:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1787828928.359333:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828928.359334:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828928.359335:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787828928.359337:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828928.359341:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298401302 00000100:00000001:0.0:1787828928.359343:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298402560 00000100:00000001:0.0:1787828928.359345:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298401302 00000100:00000001:0.0:1787828928.359347:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298402560 00000100:00000001:0.0:1787828928.359349:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787828928.359350:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828928.359352:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787828928.359354:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828928/real 1787828928] req@ffff8d30c9e2c780 x1874670646778880/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828953 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787828928.359360:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787828928.359360:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787828928.359362:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787828928.359362:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787828928.359363:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828928.359364:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828928.359365:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787828928.359368:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2c780 x1874670646778880/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828953 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787828928.359372:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828928.359373:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787828928.359374:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828928.359375:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828928.359376:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828928.359377:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787828928.359378:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787828928.359380:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828928.359382:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787828928.359384:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787828928.359386:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646778880:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787828928.359388:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1787828928.359389:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787828928.359390:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828928.359391:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787828928.359392:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787828928.359393:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828928.359395:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2d2c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646779008:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787828928.359397:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787828928.359398:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787828928.359400:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828928.359401:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1787828928.359403:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828928.359405:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787828928.359406:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417873088 : -126234291678528 : ffff8d30c9e2d2c0) 00000100:00000001:0.0:1787828928.359409:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1787828928.359414:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:0.0:1787828928.359416:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1787828928.359418:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828928.359419:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828928.359420:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787828928.359421:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828928.359423:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298401302 00000100:00000001:0.0:1787828928.359425:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298402560 00000100:00000001:0.0:1787828928.359427:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298401302 00000100:00000001:0.0:1787828928.359428:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298402560 00000100:00000001:0.0:1787828928.359430:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787828928.359431:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828928.359432:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787828928.359434:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828928/real 1787828928] req@ffff8d30c9e2d2c0 x1874670646779008/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828953 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787828928.359439:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787828928.359439:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787828928.359440:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787828928.359441:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787828928.359442:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828928.359443:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828928.359444:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787828928.359446:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2d2c0 x1874670646779008/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828953 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787828928.359455:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828928.359457:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787828928.359458:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828928.359459:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828928.359460:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828928.359461:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787828928.359462:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787828928.359463:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828928.359465:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787828928.359466:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787828928.359468:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2d2c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646779008:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787828928.359470:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1787828928.359471:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787828928.359471:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828928.359473:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787828928.359474:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787828928.359475:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828928.359476:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646779136:192.168.203.143@tcp:8:kworker.0 00000100:00000001:0.0:1787828928.359478:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787828928.359479:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787828928.359480:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828928.359481:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1787828928.359483:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828928.359486:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787828928.359487:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417871168 : -126234291680448 : ffff8d30c9e2cb40) 00000100:00000001:0.0:1787828928.359489:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1787828928.359495:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:0.0:1787828928.359497:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1787828928.359499:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828928.359500:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828928.359501:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787828928.359503:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828928.359504:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298401315 00000100:00000001:0.0:1787828928.359506:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298402560 00000100:00000001:0.0:1787828928.359508:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298401315 00000100:00000001:0.0:1787828928.359509:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298402560 00000100:00000001:0.0:1787828928.359511:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787828928.359512:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828928.359513:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787828928.359515:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828928/real 1787828928] req@ffff8d30c9e2cb40 x1874670646779136/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828953 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787828928.359520:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787828928.359521:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787828928.359522:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787828928.359523:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787828928.359523:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828928.359524:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828928.359525:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787828928.359527:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2cb40 x1874670646779136/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828953 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787828928.359531:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828928.359533:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787828928.359534:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828928.359536:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828928.359536:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828928.359538:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787828928.359539:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787828928.359540:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828928.359542:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787828928.359543:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787828928.359546:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646779136:192.168.203.143@tcp:8:kworker.0 00000100:00000001:0.0:1787828928.359549:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828928.359550:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828928.359551:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787828928.359552:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787828928.359553:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787828928.359554:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787828928.359554:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787828928.359556:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787828928.359556:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787828928.359559:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787828928.359560:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828928.359561:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828928.359561:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787828928.359562:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787828928.359564:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787828928.359565:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787828928.359565:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787828928.359566:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787828928.359567:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787828928.359570:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787828928.359570:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828928.359571:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828928.359572:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787828928.359573:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787828928.359573:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787828928.359574:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787828928.359575:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787828928.359575:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787828928.359576:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787828928.359577:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787828928.359578:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828928.359579:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828928.359580:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787828928.359581:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828928.359582:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787828928.359583:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828929.254790:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828929.254798:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828929.254869:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828929.254873:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828929.254876:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828930.278709:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828930.278718:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828930.278773:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828930.278777:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828930.278780:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828931.302800:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828931.302810:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828931.302889:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828931.302893:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828931.302897:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828932.326696:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828932.326701:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828932.326745:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828932.326749:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828932.326751:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828933.350696:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828933.350702:0:815:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1787828933.350704:0:815:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1787828933.350713:0:815:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828933.350717:0:815:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1787828933.350721:0:815:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828933.350722:0:815:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828933.350724:0:815:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828933.350800:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828933.350805:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828933.350809:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787828933.478743:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828933.478754:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1787828933.478757:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1787828933.478761:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787828933.478767:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1787828933.478775:0:40701:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 3733 00000100:00080000:1.0:1787828933.478790:0:40701:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1787828933.478793:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787828933.478796:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:1.0:1787828933.478801:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:1.0:1787828933.478804:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:1.0:1787828933.478808:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:1.0:1787828933.478811:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:1.0:1787828933.478813:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787828933.478815:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:1.0:1787828933.478819:0:40701:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:1.0:1787828933.478821:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828933.478823:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1787828933.478827:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787828933.478829:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828933.478831:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:1.0:1787828933.478833:0:40701:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828933.478842:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1787828933.478844:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828933.478847:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1787828933.478850:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1787828933.478854:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828933.478865:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787828933.478875:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d13ef0c0 x1874670646779264/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787828933.478884:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787828933.478888:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828933.478906:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1787828933.478909:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828933.478913:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1787828933.478915:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1787828933.478918:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787828933.478923:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1787828933.478928:0:40701:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3733 00000100:00080000:1.0:1787828933.478933:0:40701:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1787828933.478937:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787828933.478939:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:1.0:1787828933.478944:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:1.0:1787828933.478946:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:1.0:1787828933.478949:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000020:00000001:1.0:1787828933.478953:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:1.0:1787828933.478956:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787828933.478959:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:1.0:1787828933.478964:0:40701:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:1.0:1787828933.478968:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828933.478971:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:0.0:1787828933.478974:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787828933.478981:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 02000000:00000001:1.0:1787828933.478981:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787828933.478985:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 02000000:00000001:1.0:1787828933.478988:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828933.478990:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787828933.478992:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1787828933.478993:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828933.478999:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000008:00100000:1.0:1787828933.479000:0:40701:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 02000000:00000001:0.0:1787828933.479002:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000008:00000001:1.0:1787828933.479008:0:40701:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828933.479009:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828933.479015:0:40701:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828933.479018:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d13ef0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646779264:192.168.203.143@tcp:250:kworker.0 02000000:00000001:1.0:1787828933.479023:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:0.0:1787828933.479024:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787828933.479027:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828933.479031:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:1.0:1787828933.479032:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787828933.479035:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828933.479036:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787828933.479039:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787828933.479040:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828933.479044:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828933.479049:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787828933.479058:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d13ede00 x1874670646779392/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787828933.479062:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787828933.479065:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839541350592 : -126234168201024 : ffff8d30d13ef0c0) 00000100:00000001:0.0:1787828933.479071:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:1.0:1787828933.479073:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787828933.479077:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828933.479082:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1787828933.479085:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828933.479090:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:1.0:1787828933.479092:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1787828933.479093:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787828933.479097:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1787828933.479100:0:40701:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3733 00000100:00080000:1.0:1787828933.479104:0:40701:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1787828933.479106:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787828933.479107:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:1.0:1787828933.479109:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:1.0:1787828933.479110:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:1.0:1787828933.479114:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000800:00000100:0.0:1787828933.479115:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000020:00000001:1.0:1787828933.479119:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:1.0:1787828933.479122:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787828933.479124:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:0.0:1787828933.479128:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00080000:1.0:1787828933.479128:0:40701:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:1.0:1787828933.479131:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828933.479133:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:0.0:1787828933.479134:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 02000000:00000001:1.0:1787828933.479136:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787828933.479138:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828933.479141:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828933.479142:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787828933.479147:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1787828933.479149:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000008:00100000:1.0:1787828933.479151:0:40701:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00000001:0.0:1787828933.479154:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4298407680 00000008:00000001:1.0:1787828933.479158:0:40701:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828933.479161:0:40701:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828933.479165:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 02000000:00000001:1.0:1787828933.479168:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1787828933.479172:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828933.479173:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4298407680 00000100:00000001:1.0:1787828933.479175:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1787828933.479179:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1787828933.479181:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828933.479182:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1787828933.479188:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787828933.479192:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828933.479195:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00000001:1.0:1787828933.479197:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828933.479204:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828933/real 1787828933] req@ffff8d30d13ef0c0 x1874670646779264/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828958 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787828933.479204:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d13eed00 x1874670646779520/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000001:0.0:1787828933.479235:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000001:1.0:1787828933.479237:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000400:00000001:0.0:1787828933.479238:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787828933.479243:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828933.479244:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787828933.479247:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828933.479248:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787828933.479251:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:1.0:1787828933.479253:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1787828933.479255:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828933.479258:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:1.0:1787828933.479261:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:0.0:1787828933.479266:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d13ef0c0 x1874670646779264/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828958 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787828933.479267:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1787828933.479272:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787828933.479278:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828933.479279:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:0.0:1787828933.479283:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787828933.479286:0:40701:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3733 00000100:00000001:0.0:1787828933.479288:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828933.479292:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828933.479293:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:1.0:1787828933.479295:0:40701:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787828933.479296:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787828933.479298:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787828933.479301:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00080000:0.0:1787828933.479303:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00000001:1.0:1787828933.479306:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:0.0:1787828933.479309:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000020:00000001:1.0:1787828933.479311:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:0.0:1787828933.479313:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000020:00000001:1.0:1787828933.479316:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00100000:0.0:1787828933.479318:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d13ef0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646779264:192.168.203.143@tcp:250:kworker.0 00000020:00000001:1.0:1787828933.479320:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:0.0:1787828933.479323:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:1.0:1787828933.479327:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:0.0:1787828933.479328:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828933.479331:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 00000100:00000001:1.0:1787828933.479333:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 02000000:00000001:0.0:1787828933.479334:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 00000100:00000001:1.0:1787828933.479337:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 02000000:00000001:0.0:1787828933.479339:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787828933.479342:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787828933.479346:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 00000100:00080000:1.0:1787828933.479347:0:40701:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 02000000:00000001:0.0:1787828933.479351:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 00000100:00000001:1.0:1787828933.479352:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828933.479354:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 02000000:00000001:1.0:1787828933.479357:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:0.0:1787828933.479359:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787828933.479361:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:1.0:1787828933.479362:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787828933.479363:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:1.0:1787828933.479370:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828933.479372:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1787828933.479374:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000008:00100000:1.0:1787828933.479375:0:40701:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00000001:0.0:1787828933.479378:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:1.0:1787828933.479381:0:40701:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828933.479382:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787828933.479385:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787828933.479387:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787828933.479387:0:40701:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828933.479390:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787828933.479392:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828933.479394:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787828933.479395:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787828933.479397:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787828933.479399:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828933.479401:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828933.479405:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00100000:0.0:1787828933.479407:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d13ede00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646779392:192.168.203.143@tcp:38:kworker.0 02000000:00000001:1.0:1787828933.479410:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:0.0:1787828933.479412:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787828933.479414:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828933.479415:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787828933.479418:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828933.479420:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:1.0:1787828933.479423:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828933.479424:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828933.479434:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00100000:1.0:1787828933.479435:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d13ef840 x1874670646779648/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787828933.479437:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839541345792 : -126234168205824 : ffff8d30d13ede00) 00000100:00000001:0.0:1787828933.479444:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:1.0:1787828933.479450:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787828933.479454:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:0.0:1787828933.479459:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787828933.479461:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828933.479465:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1787828933.479469:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828933.479470:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828933.479483:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787828933.479489:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828933.479494:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298406422 00000100:00000001:0.0:1787828933.479497:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298407680 00000100:00000001:0.0:1787828933.479500:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298406422 00000100:00000001:0.0:1787828933.479502:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298407680 00000100:00000001:0.0:1787828933.479505:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787828933.479507:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828933.479509:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787828933.479514:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828933/real 1787828933] req@ffff8d30d13ede00 x1874670646779392/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828958 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787828933.479521:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787828933.479523:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787828933.479528:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787828933.479533:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787828933.479538:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828933.479541:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828933.479543:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787828933.479552:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d13ede00 x1874670646779392/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828958 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787828933.479556:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828933.479558:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787828933.479576:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828933.479579:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828933.479581:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828933.479583:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787828933.479587:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787828933.479589:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828933.479592:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787828933.479594:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787828933.479597:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d13ede00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646779392:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787828933.479719:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1787828933.479724:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787828933.479726:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828933.479730:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787828933.479732:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787828933.479735:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828933.479738:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d13eed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646779520:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787828933.479741:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787828933.479742:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787828933.479744:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828933.479746:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1787828933.479750:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828933.479757:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787828933.479758:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839541349632 : -126234168201984 : ffff8d30d13eed00) 00000100:00000001:0.0:1787828933.479762:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1787828933.479776:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:0.0:1787828933.479782:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1787828933.479785:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828933.479787:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828933.479789:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787828933.479792:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828933.479796:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298406423 00000100:00000001:0.0:1787828933.479799:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298407680 00000100:00000001:0.0:1787828933.479801:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298406423 00000100:00000001:0.0:1787828933.479803:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298407680 00000100:00000001:0.0:1787828933.479804:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787828933.479806:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828933.479807:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787828933.479810:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828933/real 1787828933] req@ffff8d30d13eed00 x1874670646779520/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828958 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787828933.479817:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787828933.479818:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787828933.479820:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787828933.479821:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787828933.479822:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828933.479823:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828933.479824:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787828933.479826:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d13eed00 x1874670646779520/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828958 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787828933.479831:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828933.479832:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787828933.479834:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828933.479835:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828933.479835:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828933.479837:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787828933.479838:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787828933.479839:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828933.479841:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787828933.479843:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787828933.479846:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d13eed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646779520:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787828933.479849:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828933.479850:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828933.479851:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787828933.479852:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787828933.479854:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787828933.479855:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787828933.479856:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787828933.479858:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787828933.479858:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787828933.479861:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787828933.479862:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828933.479863:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828933.479864:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787828933.479865:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787828933.479866:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787828933.479867:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787828933.479868:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787828933.479869:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787828933.479870:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787828933.479871:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787828933.479872:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828933.479873:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828933.479875:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787828933.479876:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828933.479877:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787828933.479878:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787828933.479879:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1787828933.479880:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787828933.479881:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828933.479882:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787828933.479883:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787828933.479885:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828933.479887:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d13ef840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646779648:192.168.203.143@tcp:8:kworker.0 00000100:00000001:0.0:1787828933.479889:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787828933.479890:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787828933.479891:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828933.479892:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1787828933.479895:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828933.479899:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787828933.479900:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839541352512 : -126234168199104 : ffff8d30d13ef840) 00000100:00000001:0.0:1787828933.479903:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1787828933.479911:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:0.0:1787828933.479914:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1787828933.479915:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828933.479916:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828933.479918:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787828933.479919:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828933.479922:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298406436 00000100:00000001:0.0:1787828933.479923:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298407680 00000100:00000001:0.0:1787828933.479925:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298406436 00000100:00000001:0.0:1787828933.479927:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298407680 00000100:00000001:0.0:1787828933.479928:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787828933.479929:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828933.479930:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787828933.479933:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828933/real 1787828933] req@ffff8d30d13ef840 x1874670646779648/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828958 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787828933.479937:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787828933.479938:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787828933.479940:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787828933.479941:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787828933.479942:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828933.479943:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828933.479944:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787828933.479946:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d13ef840 x1874670646779648/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828958 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787828933.479950:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828933.479952:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787828933.479953:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828933.479954:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828933.479955:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828933.479957:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787828933.479958:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787828933.479960:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828933.479962:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787828933.479963:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787828933.479965:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d13ef840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646779648:192.168.203.143@tcp:8:kworker.0 00000100:00000001:0.0:1787828933.479967:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828933.479968:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828933.479969:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787828933.479970:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787828933.479971:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787828933.479972:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787828933.479972:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787828933.479973:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787828933.479974:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787828933.479975:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787828933.479976:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828933.479978:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828933.479979:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787828933.479980:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828933.479982:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787828933.479983:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828934.374689:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828934.374694:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828934.374755:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828934.374760:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828934.374763:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828935.398729:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828935.398736:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828935.398812:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828935.398818:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828935.398822:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828936.422724:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828936.422731:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828936.422795:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828936.422800:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828936.422804:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828937.446718:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828937.446723:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828937.446785:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828937.446791:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828937.446795:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828938.470701:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828938.470708:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828938.470778:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828938.470783:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828938.470787:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787828938.598704:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828938.598713:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1787828938.598715:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787828938.598718:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787828938.598723:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787828938.598730:0:40701:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 3739 00000100:00080000:2.0:1787828938.598741:0:40701:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787828938.598744:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828938.598747:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787828938.598751:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787828938.598754:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787828938.598758:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:2.0:1787828938.598760:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:2.0:1787828938.598763:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828938.598765:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:2.0:1787828938.598770:0:40701:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787828938.598772:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828938.598774:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787828938.598777:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828938.598779:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828938.598781:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:2.0:1787828938.598783:0:40701:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828938.598793:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787828938.598794:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828938.598796:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787828938.598799:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787828938.598802:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828938.598812:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787828938.598819:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2d680 x1874670646779776/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787828938.598827:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787828938.598830:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828938.598843:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787828938.598845:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828938.598848:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1787828938.598849:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787828938.598850:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787828938.598852:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787828938.598854:0:40701:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3739 00000100:00080000:2.0:1787828938.598857:0:40701:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787828938.598859:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828938.598860:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787828938.598862:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787828938.598864:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787828938.598865:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000020:00000001:2.0:1787828938.598867:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:2.0:1787828938.598869:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828938.598870:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:2.0:1787828938.598872:0:40701:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787828938.598874:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828938.598875:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:2.0:1787828938.598875:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:0.0:1787828938.598881:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 02000000:00000001:2.0:1787828938.598882:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787828938.598884:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 02000000:00000001:2.0:1787828938.598885:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828938.598888:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787828938.598888:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1787828938.598892:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000008:00100000:2.0:1787828938.598895:0:40701:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 02000000:00000001:0.0:1787828938.598897:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000008:00000001:2.0:1787828938.598898:0:40701:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828938.598902:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787828938.598902:0:40701:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828938.598905:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787828938.598906:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828938.598908:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828938.598910:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787828938.598912:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00100000:0.0:1787828938.598913:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2d680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646779776:192.168.203.143@tcp:250:kworker.0 02000000:00000001:2.0:1787828938.598914:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828938.598920:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:2.0:1787828938.598921:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828938.598925:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787828938.598929:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787828938.598929:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2e1c0 x1874670646779904/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:0.0:1787828938.598935:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:2.0:1787828938.598941:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 02000000:00000001:0.0:1787828938.598942:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828938.598943:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828938.598948:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787828938.598950:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828938.598952:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1787828938.598953:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787828938.598954:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787828938.598955:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:0.0:1787828938.598957:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00080000:2.0:1787828938.598958:0:40701:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3739 00000100:00000001:0.0:1787828938.598960:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417874048 : -126234291677568 : ffff8d30c9e2d680) 00000100:00080000:2.0:1787828938.598963:0:40701:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787828938.598964:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:2.0:1787828938.598966:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828938.598968:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787828938.598970:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787828938.598971:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787828938.598972:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000020:00000001:2.0:1787828938.598974:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:2.0:1787828938.598976:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828938.598977:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:2.0:1787828938.598979:0:40701:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787828938.598980:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828938.598981:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787828938.598983:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828938.598984:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828938.598986:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1787828938.598987:0:40701:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1787828938.598989:0:40701:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828938.598990:0:40701:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828938.598994:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787828938.598994:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828938.598995:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787828938.598996:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787828938.598998:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:0.0:1787828938.599003:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:2.0:1787828938.599003:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787828938.599010:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2cb40 x1874670646780032/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787828938.599015:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787828938.599017:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828938.599019:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1787828938.599019:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787828938.599024:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1787828938.599027:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00080000:2.0:1787828938.599027:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1787828938.599031:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:0.0:1787828938.599032:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787828938.599033:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787828938.599037:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1787828938.599038:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787828938.599041:0:40701:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3739 00000100:00000001:0.0:1787828938.599045:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787828938.599045:0:40701:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787828938.599049:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828938.599051:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:0.0:1787828938.599052:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4298412800 00000020:00000001:2.0:1787828938.599053:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787828938.599055:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787828938.599058:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000020:00000001:2.0:1787828938.599059:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:2.0:1787828938.599061:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828938.599061:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:0.0:1787828938.599064:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 00000100:00080000:2.0:1787828938.599064:0:40701:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787828938.599069:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828938.599071:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:0.0:1787828938.599072:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4298412800 02000000:00000001:2.0:1787828938.599073:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828938.599076:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828938.599077:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 00000100:00000001:2.0:1787828938.599078:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1787828938.599081:0:40701:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00000001:0.0:1787828938.599083:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000008:00000001:2.0:1787828938.599084:0:40701:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828938.599089:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828938.599089:0:40701:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828938.599095:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 02000000:00000001:2.0:1787828938.599097:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787828938.599100:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828938.599102:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00100000:0.0:1787828938.599103:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828938/real 1787828938] req@ffff8d30c9e2d680 x1874670646779776/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828963 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1787828938.599104:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787828938.599107:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828938.599113:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:0.0:1787828938.599120:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00100000:2.0:1787828938.599121:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2d2c0 x1874670646780160/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000001:0.0:1787828938.599123:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787828938.599127:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1787828938.599131:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787828938.599132:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787828938.599134:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828938.599135:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828938.599139:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828938.599139:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828938.599142:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787828938.599148:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2d680 x1874670646779776/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828963 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787828938.599154:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828938.599157:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787828938.599159:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828938.599161:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828938.599162:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828938.599164:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787828938.599166:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787828938.599167:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828938.599170:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787828938.599172:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787828938.599174:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2d680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646779776:192.168.203.143@tcp:250:kworker.0 00000100:00000001:0.0:1787828938.599178:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828938.599179:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828938.599180:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787828938.599182:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787828938.599184:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787828938.599185:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787828938.599187:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787828938.599190:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787828938.599192:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787828938.599194:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787828938.599195:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828938.599197:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828938.599202:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787828938.599204:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828938.599205:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787828938.599206:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787828938.599207:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1787828938.599208:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787828938.599209:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828938.599211:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787828938.599212:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787828938.599214:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828938.599216:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646779904:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787828938.599218:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787828938.599220:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787828938.599221:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828938.599222:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1787828938.599224:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828938.599229:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787828938.599230:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417876928 : -126234291674688 : ffff8d30c9e2e1c0) 00000100:00000001:0.0:1787828938.599233:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1787828938.599243:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:0.0:1787828938.599248:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1787828938.599250:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828938.599251:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828938.599253:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787828938.599256:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828938.599261:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298411542 00000100:00000001:0.0:1787828938.599264:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298412800 00000100:00000001:0.0:1787828938.599266:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298411542 00000100:00000001:0.0:1787828938.599268:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298412800 00000100:00000001:0.0:1787828938.599271:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787828938.599273:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828938.599275:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787828938.599280:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828938/real 1787828938] req@ffff8d30c9e2e1c0 x1874670646779904/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828963 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787828938.599290:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787828938.599291:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787828938.599294:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787828938.599296:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787828938.599297:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828938.599299:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828938.599300:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787828938.599303:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2e1c0 x1874670646779904/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828963 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787828938.599317:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828938.599318:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787828938.599320:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828938.599321:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828938.599322:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828938.599324:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787828938.599326:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787828938.599328:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828938.599330:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787828938.599332:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787828938.599334:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646779904:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787828938.599336:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1787828938.599337:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787828938.599338:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828938.599339:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787828938.599341:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787828938.599342:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828938.599344:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646780032:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787828938.599347:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787828938.599348:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787828938.599349:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828938.599350:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1787828938.599352:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828938.599356:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787828938.599357:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417871168 : -126234291680448 : ffff8d30c9e2cb40) 00000100:00000001:0.0:1787828938.599360:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1787828938.599369:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:0.0:1787828938.599372:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1787828938.599374:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828938.599375:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828938.599376:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787828938.599378:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828938.599380:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298411542 00000100:00000001:0.0:1787828938.599382:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298412800 00000100:00000001:0.0:1787828938.599384:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298411542 00000100:00000001:0.0:1787828938.599386:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298412800 00000100:00000001:0.0:1787828938.599389:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787828938.599390:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828938.599392:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787828938.599394:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828938/real 1787828938] req@ffff8d30c9e2cb40 x1874670646780032/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828963 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787828938.599400:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787828938.599401:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787828938.599402:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787828938.599404:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787828938.599404:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828938.599405:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828938.599406:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787828938.599409:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2cb40 x1874670646780032/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828963 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787828938.599414:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828938.599415:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787828938.599417:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828938.599418:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828938.599418:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828938.599419:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787828938.599420:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787828938.599422:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828938.599423:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787828938.599425:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787828938.599426:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646780032:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787828938.599429:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1787828938.599429:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787828938.599430:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828938.599431:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787828938.599432:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787828938.599434:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828938.599436:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2d2c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646780160:192.168.203.143@tcp:8:kworker.0 00000100:00000001:0.0:1787828938.599438:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787828938.599439:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787828938.599441:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828938.599442:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1787828938.599444:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828938.599447:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787828938.599447:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417873088 : -126234291678528 : ffff8d30c9e2d2c0) 00000100:00000001:0.0:1787828938.599450:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1787828938.599455:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:0.0:1787828938.599457:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1787828938.599458:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828938.599460:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828938.599461:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787828938.599462:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828938.599464:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298411555 00000100:00000001:0.0:1787828938.599466:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298412800 00000100:00000001:0.0:1787828938.599467:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298411555 00000100:00000001:0.0:1787828938.599469:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298412800 00000100:00000001:0.0:1787828938.599470:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787828938.599472:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828938.599473:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787828938.599475:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828938/real 1787828938] req@ffff8d30c9e2d2c0 x1874670646780160/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828963 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787828938.599480:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787828938.599481:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787828938.599482:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787828938.599483:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787828938.599484:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828938.599485:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828938.599485:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787828938.599488:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2d2c0 x1874670646780160/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828963 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787828938.599498:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828938.599499:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787828938.599501:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828938.599502:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828938.599502:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828938.599503:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787828938.599504:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787828938.599506:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828938.599507:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787828938.599508:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787828938.599510:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2d2c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646780160:192.168.203.143@tcp:8:kworker.0 00000100:00000001:0.0:1787828938.599512:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828938.599514:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828938.599514:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787828938.599515:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787828938.599517:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787828938.599517:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787828938.599518:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787828938.599520:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787828938.599520:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787828938.599523:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787828938.599535:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828938.599538:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828938.599539:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787828938.599539:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787828938.599541:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787828938.599541:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787828938.599542:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787828938.599543:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787828938.599544:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787828938.599545:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787828938.599546:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828938.599547:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828938.599548:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787828938.599549:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787828938.599550:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787828938.599551:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787828938.599552:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787828938.599553:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787828938.599553:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787828938.599555:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787828938.599556:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828938.599557:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828938.599559:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787828938.599560:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828938.599561:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787828938.599562:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828939.494755:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828939.494764:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828939.494880:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828939.494886:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828939.494890:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828940.518689:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828940.518696:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828940.518745:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828940.518749:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828940.518753:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828941.542734:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828941.542742:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828941.542833:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828941.542841:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828941.542848:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828942.566738:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828942.566748:0:815:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1787828942.566751:0:815:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1787828942.566755:0:815:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828942.566762:0:815:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1787828942.566765:0:815:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828942.566767:0:815:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828942.566770:0:815:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828942.566869:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828942.566876:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828942.566880:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828943.590741:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828943.590748:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828943.590820:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828943.590826:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828943.590830:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787828943.718764:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828943.718776:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1787828943.718779:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787828943.718784:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787828943.718790:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787828943.718799:0:468:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 3744 00000100:00080000:3.0:1787828943.718812:0:468:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787828943.718816:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787828943.718818:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:3.0:1787828943.718824:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1787828943.718827:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1787828943.718831:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:3.0:1787828943.718834:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:3.0:1787828943.718837:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787828943.718838:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:3.0:1787828943.718844:0:468:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:3.0:1787828943.718847:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787828943.718850:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787828943.718854:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787828943.718858:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828943.718861:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:3.0:1787828943.718864:0:468:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787828943.718877:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787828943.718879:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828943.718883:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787828943.718886:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787828943.718891:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828943.718906:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787828943.718930:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c7017c00 x1874670646780288/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787828943.718953:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787828943.718958:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828943.718976:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787828943.718980:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828943.718984:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1787828943.718986:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787828943.718988:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787828943.718992:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787828943.718995:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3744 00000100:00080000:3.0:1787828943.718999:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787828943.719001:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787828943.719003:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:3.0:1787828943.719006:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1787828943.719008:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1787828943.719010:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000020:00000001:3.0:1787828943.719014:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:3.0:1787828943.719017:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787828943.719018:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:3.0:1787828943.719023:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:3.0:1787828943.719026:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787828943.719029:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787828943.719036:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787828943.719039:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828943.719042:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:3.0:1787828943.719048:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00000001:0.0:1787828943.719053:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000008:00000001:3.0:1787828943.719053:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828943.719062:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828943.719066:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 02000000:00000001:3.0:1787828943.719069:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787828943.719075:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828943.719076:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787828943.719078:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787828943.719084:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:0.0:1787828943.719086:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 02000000:00000001:3.0:1787828943.719088:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828943.719095:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828943.719101:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828943.719107:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000100:00100000:3.0:1787828943.719112:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c7016940 x1874670646780416/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:0.0:1787828943.719116:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787828943.719129:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828943.719135:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787828943.719141:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828943.719147:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c7017c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646780288:192.168.203.143@tcp:250:kworker.0 00000100:00000001:3.0:1787828943.719148:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787828943.719156:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1787828943.719168:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00080000:3.0:1787828943.719169:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1787828943.719177:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787828943.719179:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 02000000:00000001:0.0:1787828943.719180:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:3.0:1787828943.719183:0:468:0:(import.c:485:import_select_connection()) Process entered 02000000:00000001:0.0:1787828943.719191:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787828943.719192:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3744 00000100:00080000:3.0:1787828943.719201:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 02000000:00000001:0.0:1787828943.719204:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:3.0:1787828943.719206:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787828943.719211:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:3.0:1787828943.719213:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1787828943.719214:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 02000000:00000001:0.0:1787828943.719217:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828943.719218:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000020:00000001:3.0:1787828943.719228:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:3.0:1787828943.719231:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787828943.719234:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:3.0:1787828943.719238:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:3.0:1787828943.719241:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787828943.719243:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:0.0:1787828943.719248:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 02000000:00000001:3.0:1787828943.719249:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787828943.719257:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369550848 : -126234340000768 : ffff8d30c7017c00) 02000000:00000001:3.0:1787828943.719258:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828943.719264:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:3.0:1787828943.719270:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00000001:0.0:1787828943.719272:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000008:00000001:3.0:1787828943.719277:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828943.719282:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787828943.719288:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787828943.719290:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828943.719292:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787828943.719294:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787828943.719296:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828943.719303:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787828943.719308:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c7014000 x1874670646780544/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787828943.719319:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787828943.719322:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828943.719325:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787828943.719329:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:0.0:1787828943.719340:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00080000:3.0:1787828943.719341:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1787828943.719350:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787828943.719353:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787828943.719356:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787828943.719361:0:468:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3744 00000100:00000001:0.0:1787828943.719367:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00080000:3.0:1787828943.719367:0:468:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787828943.719376:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787828943.719378:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:0.0:1787828943.719381:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000020:00000001:3.0:1787828943.719382:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1787828943.719389:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:0.0:1787828943.719392:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828943.719393:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:0.0:1787828943.719402:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000020:00000001:3.0:1787828943.719403:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:3.0:1787828943.719410:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787828943.719413:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:0.0:1787828943.719417:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787828943.719419:0:468:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:3.0:1787828943.719428:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828943.719432:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4298417920 02000000:00000001:3.0:1787828943.719432:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787828943.719438:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787828943.719443:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828943.719445:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:3.0:1787828943.719448:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1787828943.719452:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828943.719454:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 00000100:00000001:3.0:1787828943.719455:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787828943.719464:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787828943.719467:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828943.719470:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787828943.719472:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:0.0:1787828943.719474:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4298417920 02000000:00000001:3.0:1787828943.719475:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828943.719483:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787828943.719515:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c7017840 x1874670646780672/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787828943.719518:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1787828943.719533:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787828943.719534:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787828943.719541:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828943.719546:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828943.719560:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828943.719570:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787828943.719586:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828943/real 1787828943] req@ffff8d30c7017c00 x1874670646780288/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828968 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787828943.719692:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787828943.719696:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787828943.719703:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787828943.719708:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787828943.719712:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828943.719717:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828943.719721:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787828943.719732:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c7017c00 x1874670646780288/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828968 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787828943.719748:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828943.719755:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787828943.719761:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828943.719765:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828943.719768:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828943.719773:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787828943.719777:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787828943.719783:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828943.719791:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787828943.719797:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787828943.719805:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c7017c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646780288:192.168.203.143@tcp:250:kworker.0 00000100:00000001:0.0:1787828943.719815:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828943.719820:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828943.719823:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787828943.719828:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787828943.719835:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787828943.719839:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787828943.719863:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787828943.719871:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787828943.719874:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787828943.719879:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787828943.719881:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828943.719887:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828943.719898:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787828943.719902:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828943.719907:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787828943.719910:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787828943.719976:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1787828943.719980:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787828943.719983:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828943.719989:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787828943.719994:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787828943.720000:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828943.720010:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c7016940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646780416:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787828943.720019:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787828943.720024:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787828943.720028:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828943.720033:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1787828943.720040:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828943.720055:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787828943.720059:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369546048 : -126234340005568 : ffff8d30c7016940) 00000100:00000001:0.0:1787828943.720069:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1787828943.720099:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:0.0:1787828943.720110:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1787828943.720115:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828943.720118:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828943.720123:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787828943.720129:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828943.720137:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298416663 00000100:00000001:0.0:1787828943.720144:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298417920 00000100:00000001:0.0:1787828943.720149:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298416663 00000100:00000001:0.0:1787828943.720153:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298417920 00000100:00000001:0.0:1787828943.720159:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787828943.720164:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828943.720169:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787828943.720179:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828943/real 1787828943] req@ffff8d30c7016940 x1874670646780416/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828968 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787828943.720198:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787828943.720201:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787828943.720207:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787828943.720210:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787828943.720213:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828943.720218:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828943.720221:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787828943.720229:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c7016940 x1874670646780416/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828968 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787828943.720246:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828943.720251:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787828943.720256:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828943.720260:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828943.720263:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828943.720267:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787828943.720271:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787828943.720277:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828943.720284:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787828943.720289:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787828943.720297:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c7016940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646780416:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787828943.720305:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1787828943.720308:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787828943.720311:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828943.720315:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787828943.720319:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787828943.720323:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828943.720330:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c7014000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646780544:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787828943.720337:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787828943.720341:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787828943.720345:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828943.720349:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1787828943.720355:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828943.720366:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787828943.720370:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369535488 : -126234340016128 : ffff8d30c7014000) 00000100:00000001:0.0:1787828943.720376:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1787828943.720401:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:0.0:1787828943.720410:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1787828943.720415:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828943.720418:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828943.720423:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787828943.720428:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828943.720435:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298416663 00000100:00000001:0.0:1787828943.720441:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298417920 00000100:00000001:0.0:1787828943.720446:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298416663 00000100:00000001:0.0:1787828943.720451:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298417920 00000100:00000001:0.0:1787828943.720455:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787828943.720477:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828943.720482:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787828943.720515:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828943/real 1787828943] req@ffff8d30c7014000 x1874670646780544/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828968 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787828943.720529:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787828943.720531:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787828943.720536:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787828943.720538:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787828943.720540:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828943.720542:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828943.720545:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787828943.720551:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c7014000 x1874670646780544/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828968 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787828943.720568:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828943.720571:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787828943.720574:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828943.720576:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828943.720578:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828943.720580:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787828943.720582:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787828943.720585:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828943.720589:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787828943.720591:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787828943.720595:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c7014000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646780544:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787828943.720626:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1787828943.720629:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787828943.720631:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828943.720634:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787828943.720636:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787828943.720639:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828943.720643:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c7017840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646780672:192.168.203.143@tcp:8:kworker.0 00000100:00000001:0.0:1787828943.720648:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787828943.720651:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787828943.720653:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828943.720655:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1787828943.720660:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828943.720668:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787828943.720669:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369549888 : -126234340001728 : ffff8d30c7017840) 00000100:00000001:0.0:1787828943.720674:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1787828943.720691:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:0.0:1787828943.720697:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1787828943.720701:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828943.720703:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828943.720705:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787828943.720709:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828943.720713:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298416677 00000100:00000001:0.0:1787828943.720716:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298417920 00000100:00000001:0.0:1787828943.720719:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298416677 00000100:00000001:0.0:1787828943.720722:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298417920 00000100:00000001:0.0:1787828943.720724:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787828943.720727:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828943.720729:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787828943.720734:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828943/real 1787828943] req@ffff8d30c7017840 x1874670646780672/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828968 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787828943.720743:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787828943.720744:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787828943.720747:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787828943.720749:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787828943.720751:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828943.720753:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828943.720755:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787828943.720759:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c7017840 x1874670646780672/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828968 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787828943.720767:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828943.720770:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787828943.720772:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828943.720774:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828943.720776:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828943.720778:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787828943.720780:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787828943.720783:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828943.720788:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787828943.720790:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787828943.720795:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c7017840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646780672:192.168.203.143@tcp:8:kworker.0 00000100:00000001:0.0:1787828943.720800:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828943.720803:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828943.720804:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787828943.720807:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787828943.720809:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787828943.720811:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787828943.720813:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787828943.720816:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787828943.720818:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787828943.720820:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787828943.720822:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828943.720824:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828943.720826:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787828943.720827:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787828943.720830:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787828943.720831:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787828943.720833:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787828943.720834:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787828943.720836:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787828943.720838:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787828943.720839:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828943.720841:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828943.720842:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787828943.720844:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787828943.720845:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787828943.720846:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787828943.720848:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787828943.720849:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787828943.720851:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787828943.720852:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787828943.720854:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828943.720856:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828943.720861:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787828943.720863:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828943.720865:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787828943.720867:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828944.614675:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828944.614681:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828944.614733:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828944.614737:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828944.614740:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828945.638728:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828945.638736:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828945.638807:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828945.638813:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828945.638817:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828946.662698:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828946.662703:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828946.662753:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828946.662757:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828946.662759:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828947.686661:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828947.686666:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828947.686707:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828947.686710:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828947.686713:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828948.710709:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828948.710717:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828948.710786:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828948.710789:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828948.710792:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787828948.838704:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828948.838713:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1787828948.838715:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1787828948.838719:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787828948.838723:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1787828948.838730:0:468:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 3749 00000100:00080000:1.0:1787828948.838755:0:468:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1787828948.838758:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787828948.838759:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:1.0:1787828948.838762:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:1.0:1787828948.838764:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:1.0:1787828948.838767:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:1.0:1787828948.838768:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:1.0:1787828948.838770:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787828948.838771:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:1.0:1787828948.838775:0:468:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:1.0:1787828948.838777:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828948.838778:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1787828948.838781:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787828948.838784:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828948.838785:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:1.0:1787828948.838787:0:468:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828948.838796:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1787828948.838798:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828948.838801:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1787828948.838803:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1787828948.838807:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828948.838816:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787828948.838823:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d13ef480 x1874670646780800/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787828948.838831:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787828948.838834:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828948.838846:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1787828948.838848:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828948.838851:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1787828948.838852:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1787828948.838853:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787828948.838855:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1787828948.838857:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3749 00000100:00080000:1.0:1787828948.838859:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1787828948.838860:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787828948.838861:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:1.0:1787828948.838863:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:1.0:1787828948.838863:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:1.0:1787828948.838865:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000020:00000001:1.0:1787828948.838866:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:1.0:1787828948.838868:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787828948.838869:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:1.0:1787828948.838871:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:1.0:1787828948.838872:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828948.838873:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1787828948.838878:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787828948.838879:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828948.838880:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:1.0:1787828948.838884:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1787828948.838886:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828948.838888:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828948.838890:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1787828948.838891:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828948.838900:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1787828948.838901:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1787828948.838903:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828948.838908:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787828948.838912:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d13ee580 x1874670646780928/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787828948.838919:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787828948.838921:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828948.838923:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1787828948.838925:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828948.838928:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:1.0:1787828948.838929:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1787828948.838931:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787828948.838932:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1787828948.838934:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3749 00000100:00080000:1.0:1787828948.838936:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1787828948.838938:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787828948.838939:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:0.0:1787828948.838940:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:1.0:1787828948.838942:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:1.0:1787828948.838948:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:1.0:1787828948.838951:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:0.0:1787828948.838952:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000020:00000001:1.0:1787828948.838956:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:0.0:1787828948.838958:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787828948.838960:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787828948.838963:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787828948.838966:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828948.838967:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:1.0:1787828948.838974:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 02000000:00000001:0.0:1787828948.838975:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787828948.838984:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787828948.838984:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828948.838993:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787828948.838995:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828948.838996:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787828948.839004:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828948.839005:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:1.0:1787828948.839007:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1787828948.839012:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828948.839015:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828948.839016:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d13ef480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646780800:192.168.203.143@tcp:250:kworker.0 02000000:00000001:1.0:1787828948.839018:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1787828948.839021:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828948.839025:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1787828948.839030:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:0.0:1787828948.839031:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787828948.839033:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828948.839038:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828948.839041:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000100:00100000:1.0:1787828948.839045:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d13ee1c0 x1874670646781056/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:0.0:1787828948.839047:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828948.839054:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:1.0:1787828948.839064:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 02000000:00000001:0.0:1787828948.839065:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828948.839073:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828948.839081:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1787828948.839086:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1787828948.839090:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00080000:1.0:1787828948.839090:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1787828948.839092:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839541351552 : -126234168200064 : ffff8d30d13ef480) 00000100:00000001:1.0:1787828948.839095:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:0.0:1787828948.839098:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00080000:1.0:1787828948.839100:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787828948.839105:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1787828948.839109:0:468:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3749 00000100:00080000:1.0:1787828948.839133:0:468:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1787828948.839137:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787828948.839138:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:1.0:1787828948.839141:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000800:00000100:0.0:1787828948.839147:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000020:00000001:1.0:1787828948.839148:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:1.0:1787828948.839153:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000020:00000001:1.0:1787828948.839157:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:1.0:1787828948.839160:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787828948.839163:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787828948.839163:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:0.0:1787828948.839170:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00080000:1.0:1787828948.839172:0:468:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787828948.839174:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828948.839181:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787828948.839182:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828948.839187:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:0.0:1787828948.839189:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828948.839197:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787828948.839201:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828948.839202:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1787828948.839204:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4298423040 00000008:00100000:1.0:1787828948.839204:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000008:00000001:1.0:1787828948.839214:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828948.839216:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828948.839220:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1787828948.839221:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828948.839222:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 00000100:00000001:1.0:1787828948.839223:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1787828948.839230:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1787828948.839234:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828948.839237:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4298423040 00000100:00000001:1.0:1787828948.839241:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828948.839244:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1787828948.839249:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00100000:1.0:1787828948.839251:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d13eda40 x1874670646781184/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787828948.839253:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828948.839257:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787828948.839268:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828948/real 1787828948] req@ffff8d30d13ef480 x1874670646780800/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828973 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787828948.839281:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787828948.839289:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:0.0:1787828948.839296:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000001:1.0:1787828948.839297:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:0.0:1787828948.839298:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787828948.839302:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787828948.839306:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787828948.839308:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828948.839311:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828948.839314:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787828948.839319:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d13ef480 x1874670646780800/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828973 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787828948.839326:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828948.839329:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787828948.839332:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828948.839333:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828948.839333:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828948.839336:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787828948.839337:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787828948.839340:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828948.839343:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787828948.839344:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787828948.839347:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d13ef480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646780800:192.168.203.143@tcp:250:kworker.0 00000100:00000001:0.0:1787828948.839351:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1787828948.839352:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787828948.839353:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828948.839355:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787828948.839356:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787828948.839358:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828948.839360:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d13ee580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646780928:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787828948.839363:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787828948.839364:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787828948.839365:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828948.839366:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1787828948.839369:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828948.839373:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787828948.839374:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839541347712 : -126234168203904 : ffff8d30d13ee580) 00000100:00000001:0.0:1787828948.839379:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1787828948.839387:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:0.0:1787828948.839390:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1787828948.839392:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828948.839393:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828948.839394:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787828948.839396:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828948.839400:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298421782 00000100:00000001:0.0:1787828948.839402:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298423040 00000100:00000001:0.0:1787828948.839403:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298421782 00000100:00000001:0.0:1787828948.839405:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298423040 00000100:00000001:0.0:1787828948.839406:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787828948.839408:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828948.839409:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787828948.839411:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828948/real 1787828948] req@ffff8d30d13ee580 x1874670646780928/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828973 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787828948.839416:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787828948.839416:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787828948.839418:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787828948.839419:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787828948.839420:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828948.839421:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828948.839422:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787828948.839424:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d13ee580 x1874670646780928/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828973 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787828948.839428:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828948.839429:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787828948.839431:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828948.839432:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828948.839432:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828948.839433:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787828948.839435:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787828948.839436:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828948.839438:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787828948.839440:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787828948.839442:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d13ee580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646780928:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787828948.839446:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828948.839447:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828948.839448:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787828948.839451:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787828948.839470:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787828948.839472:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787828948.839474:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787828948.839478:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787828948.839479:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787828948.839482:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787828948.839483:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828948.839485:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828948.839486:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787828948.839487:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787828948.839488:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787828948.839489:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787828948.839489:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787828948.839491:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787828948.839492:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787828948.839493:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787828948.839494:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828948.839496:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828948.839504:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787828948.839505:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828948.839507:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787828948.839508:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787828948.839509:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1787828948.839510:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787828948.839511:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828948.839513:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787828948.839514:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787828948.839516:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828948.839518:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d13ee1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646781056:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787828948.839521:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787828948.839523:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787828948.839524:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828948.839526:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1787828948.839527:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828948.839531:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787828948.839532:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839541346752 : -126234168204864 : ffff8d30d13ee1c0) 00000100:00000001:0.0:1787828948.839536:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1787828948.839543:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:0.0:1787828948.839547:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1787828948.839548:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828948.839549:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828948.839550:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787828948.839553:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828948.839556:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298421782 00000100:00000001:0.0:1787828948.839559:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298423040 00000100:00000001:0.0:1787828948.839561:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298421782 00000100:00000001:0.0:1787828948.839564:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298423040 00000100:00000001:0.0:1787828948.839566:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787828948.839568:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828948.839570:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787828948.839574:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828948/real 1787828948] req@ffff8d30d13ee1c0 x1874670646781056/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828973 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787828948.839583:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787828948.839584:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787828948.839587:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787828948.839589:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787828948.839590:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828948.839592:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828948.839594:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787828948.839598:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d13ee1c0 x1874670646781056/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828973 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787828948.839643:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828948.839644:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787828948.839647:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828948.839648:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828948.839649:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828948.839651:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787828948.839652:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787828948.839654:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828948.839656:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787828948.839658:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787828948.839660:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d13ee1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646781056:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787828948.839663:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1787828948.839664:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787828948.839665:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828948.839667:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787828948.839668:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787828948.839670:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828948.839672:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d13eda40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646781184:192.168.203.143@tcp:8:kworker.0 00000100:00000001:0.0:1787828948.839675:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787828948.839676:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787828948.839678:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828948.839679:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1787828948.839681:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828948.839686:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787828948.839686:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839541344832 : -126234168206784 : ffff8d30d13eda40) 00000100:00000001:0.0:1787828948.839689:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1787828948.839698:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:0.0:1787828948.839701:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1787828948.839703:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828948.839704:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828948.839705:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787828948.839707:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828948.839710:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298421796 00000100:00000001:0.0:1787828948.839711:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298423040 00000100:00000001:0.0:1787828948.839714:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298421796 00000100:00000001:0.0:1787828948.839715:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298423040 00000100:00000001:0.0:1787828948.839717:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787828948.839719:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828948.839720:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787828948.839722:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828948/real 1787828948] req@ffff8d30d13eda40 x1874670646781184/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828973 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787828948.839727:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787828948.839728:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787828948.839729:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787828948.839730:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787828948.839731:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828948.839733:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828948.839734:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787828948.839736:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d13eda40 x1874670646781184/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828973 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787828948.839740:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828948.839741:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787828948.839743:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828948.839743:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828948.839744:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828948.839745:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787828948.839746:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787828948.839748:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828948.839749:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787828948.839751:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787828948.839753:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d13eda40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646781184:192.168.203.143@tcp:8:kworker.0 00000100:00000001:0.0:1787828948.839755:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828948.839757:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828948.839759:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787828948.839760:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787828948.839761:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787828948.839762:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787828948.839763:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787828948.839765:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787828948.839766:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787828948.839768:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787828948.839769:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828948.839770:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828948.839771:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787828948.839772:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787828948.839773:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787828948.839773:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787828948.839774:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787828948.839776:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787828948.839777:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787828948.839779:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787828948.839779:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828948.839780:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828948.839782:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787828948.839783:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828948.839784:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787828948.839786:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828949.734686:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828949.734692:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828949.734758:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828949.734762:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828949.734766:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828950.758682:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828950.758686:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828950.758727:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828950.758730:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828950.758732:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828951.782702:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828951.782707:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828951.782754:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828951.782757:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828951.782759:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828952.806697:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828952.806704:0:815:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1787828952.806705:0:815:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1787828952.806708:0:815:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828952.806712:0:815:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1787828952.806714:0:815:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828952.806716:0:815:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828952.806717:0:815:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828952.806776:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828952.806780:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828952.806783:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828953.830700:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828953.830707:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828953.830759:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828953.830901:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828953.830906:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787828953.958696:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828953.958705:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1787828953.958706:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1787828953.958726:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787828953.958729:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1787828953.958736:0:468:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 3754 00000100:00080000:1.0:1787828953.958745:0:468:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1787828953.958747:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787828953.958749:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:1.0:1787828953.958752:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:1.0:1787828953.958754:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:1.0:1787828953.958763:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:1.0:1787828953.958765:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:1.0:1787828953.958766:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787828953.958767:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:1.0:1787828953.958771:0:468:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:1.0:1787828953.958773:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828953.958790:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1787828953.958793:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787828953.958795:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828953.958797:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:1.0:1787828953.958799:0:468:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828953.958809:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1787828953.958810:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828953.958812:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1787828953.958815:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1787828953.958818:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828953.958827:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787828953.958835:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d13ecf00 x1874670646781312/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787828953.958842:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787828953.958846:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828953.958857:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1787828953.958859:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828953.958861:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1787828953.958862:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1787828953.958863:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787828953.958866:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1787828953.958867:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3754 00000100:00080000:1.0:1787828953.958869:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1787828953.958871:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787828953.958872:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:1.0:1787828953.958873:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:1.0:1787828953.958874:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:1.0:1787828953.958875:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000020:00000001:1.0:1787828953.958877:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:1.0:1787828953.958878:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787828953.958879:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:1.0:1787828953.958881:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:1.0:1787828953.958882:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828953.958883:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1787828953.958887:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787828953.958888:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828953.958890:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:1.0:1787828953.958893:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1787828953.958895:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828953.958896:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828953.958900:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1787828953.958901:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828953.958902:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1787828953.958903:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1787828953.958904:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828953.958924:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787828953.958929:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d13eda40 x1874670646781440/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787828953.958934:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787828953.958936:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828953.958938:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1787828953.958946:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1787828953.958949:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00080000:1.0:1787828953.958949:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:1.0:1787828953.958953:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1787828953.958955:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787828953.958956:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1787828953.958958:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:0.0:1787828953.958959:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00080000:1.0:1787828953.958963:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3754 00000100:00000001:0.0:1787828953.958964:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787828953.958968:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1787828953.958969:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1787828953.958973:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 02000000:00000001:0.0:1787828953.958974:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:1.0:1787828953.958974:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:1.0:1787828953.958978:0:468:0:(genops.c:792:class_conn2export()) Process entered 02000000:00000001:0.0:1787828953.958979:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000020:00000001:1.0:1787828953.958980:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:1.0:1787828953.958983:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 02000000:00000001:0.0:1787828953.958984:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787828953.958987:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:1.0:1787828953.958990:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00100000:0.0:1787828953.958992:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d13ecf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646781312:192.168.203.143@tcp:250:kworker.0 00000100:00000001:1.0:1787828953.958993:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:0.0:1787828953.959001:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00080000:1.0:1787828953.959001:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:1.0:1787828953.959006:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828953.959007:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787828953.959008:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787828953.959029:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828953.959032:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787828953.959034:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787828953.959037:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828953.959038:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828953.959046:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:1.0:1787828953.959048:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1787828953.959051:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828953.959052:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828953.959056:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 02000000:00000001:1.0:1787828953.959057:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:0.0:1787828953.959059:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839541341952 : -126234168209664 : ffff8d30d13ecf00) 02000000:00000001:1.0:1787828953.959060:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828953.959066:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:0.0:1787828953.959068:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 02000000:00000001:1.0:1787828953.959069:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1787828953.959072:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828953.959093:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787828953.959097:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d13ee1c0 x1874670646781568/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787828953.959102:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787828953.959103:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828953.959106:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1787828953.959108:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828953.959112:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1787828953.959113:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1787828953.959115:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787828953.959118:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1787828953.959124:0:468:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3754 00000800:00000100:0.0:1787828953.959125:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00080000:1.0:1787828953.959131:0:468:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1787828953.959135:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787828953.959137:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787828953.959137:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:1.0:1787828953.959142:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:0.0:1787828953.959143:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000020:00000001:1.0:1787828953.959146:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:0.0:1787828953.959147:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828953.959151:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000020:00000001:1.0:1787828953.959152:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:0.0:1787828953.959155:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787828953.959157:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:0.0:1787828953.959159:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4298428160 00000100:00000001:1.0:1787828953.959162:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787828953.959164:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:1.0:1787828953.959168:0:468:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787828953.959169:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 00000100:00000001:1.0:1787828953.959170:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828953.959173:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:0.0:1787828953.959177:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4298428160 02000000:00000001:1.0:1787828953.959177:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787828953.959183:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828953.959184:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 00000100:00000001:1.0:1787828953.959185:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:1.0:1787828953.959189:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00000001:0.0:1787828953.959190:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000008:00000001:1.0:1787828953.959193:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828953.959194:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828953.959197:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828953.959198:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 02000000:00000001:1.0:1787828953.959202:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00100000:0.0:1787828953.959205:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828953/real 1787828953] req@ffff8d30d13ecf00 x1874670646781312/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828978 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:1.0:1787828953.959205:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828953.959210:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1787828953.959212:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1787828953.959215:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:0.0:1787828953.959223:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000001:1.0:1787828953.959223:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:0.0:1787828953.959225:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787828953.959230:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00100000:1.0:1787828953.959231:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d13ee580 x1874670646781696/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787828953.959232:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787828953.959235:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828953.959240:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828953.959243:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828953.959244:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787828953.959247:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828953.959250:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828953.959251:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d13ecf00 x1874670646781312/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828978 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787828953.959258:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828953.959262:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787828953.959264:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828953.959265:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828953.959266:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828953.959268:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787828953.959270:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787828953.959272:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828953.959276:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787828953.959278:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787828953.959281:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d13ecf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646781312:192.168.203.143@tcp:250:kworker.0 00000100:00000001:0.0:1787828953.959284:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1787828953.959285:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787828953.959286:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828953.959287:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787828953.959289:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787828953.959290:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828953.959292:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d13eda40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646781440:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787828953.959294:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787828953.959295:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787828953.959296:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828953.959298:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1787828953.959300:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828953.959304:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787828953.959304:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839541344832 : -126234168206784 : ffff8d30d13eda40) 00000100:00000001:0.0:1787828953.959309:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1787828953.959316:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:0.0:1787828953.959318:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1787828953.959320:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828953.959321:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828953.959322:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787828953.959323:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828953.959326:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298426902 00000100:00000001:0.0:1787828953.959328:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298428160 00000100:00000001:0.0:1787828953.959330:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298426902 00000100:00000001:0.0:1787828953.959331:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298428160 00000100:00000001:0.0:1787828953.959333:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787828953.959334:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828953.959335:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787828953.959337:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828953/real 1787828953] req@ffff8d30d13eda40 x1874670646781440/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828978 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787828953.959344:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787828953.959345:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787828953.959346:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787828953.959347:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787828953.959348:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828953.959349:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828953.959350:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787828953.959352:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d13eda40 x1874670646781440/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828978 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787828953.959356:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828953.959358:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787828953.959359:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828953.959360:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828953.959361:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828953.959362:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787828953.959363:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787828953.959365:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828953.959367:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787828953.959368:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787828953.959370:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d13eda40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646781440:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787828953.959372:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828953.959374:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828953.959375:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787828953.959376:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787828953.959378:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787828953.959380:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787828953.959381:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787828953.959384:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787828953.959385:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787828953.959387:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787828953.959388:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828953.959390:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828953.959390:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787828953.959391:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787828953.959392:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787828953.959393:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787828953.959394:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787828953.959395:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787828953.959396:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787828953.959397:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787828953.959397:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828953.959399:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828953.959405:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787828953.959406:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828953.959407:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787828953.959408:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787828953.959409:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1787828953.959410:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787828953.959411:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828953.959412:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787828953.959413:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787828953.959414:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828953.959416:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d13ee1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646781568:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787828953.959438:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787828953.959440:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787828953.959442:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828953.959444:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1787828953.959447:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828953.959451:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787828953.959453:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839541346752 : -126234168204864 : ffff8d30d13ee1c0) 00000100:00000001:0.0:1787828953.959471:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1787828953.959485:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:0.0:1787828953.959490:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1787828953.959493:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828953.959494:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828953.959496:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787828953.959498:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828953.959501:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298426902 00000100:00000001:0.0:1787828953.959503:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298428160 00000100:00000001:0.0:1787828953.959505:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298426902 00000100:00000001:0.0:1787828953.959507:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298428160 00000100:00000001:0.0:1787828953.959508:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787828953.959510:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828953.959511:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787828953.959514:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828953/real 1787828953] req@ffff8d30d13ee1c0 x1874670646781568/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828978 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787828953.959519:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787828953.959520:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787828953.959522:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787828953.959523:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787828953.959524:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828953.959526:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828953.959527:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787828953.959529:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d13ee1c0 x1874670646781568/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828978 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787828953.959534:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828953.959535:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787828953.959537:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828953.959538:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828953.959539:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828953.959540:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787828953.959541:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787828953.959543:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828953.959545:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787828953.959546:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787828953.959548:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d13ee1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646781568:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787828953.959551:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1787828953.959552:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787828953.959553:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828953.959555:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787828953.959564:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787828953.959566:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828953.959584:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d13ee580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646781696:192.168.203.143@tcp:8:kworker.0 00000100:00000001:0.0:1787828953.959587:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787828953.959588:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787828953.959589:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828953.959591:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1787828953.959593:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828953.959597:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787828953.959598:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839541347712 : -126234168203904 : ffff8d30d13ee580) 00000100:00000001:0.0:1787828953.959600:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1787828953.959608:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:0.0:1787828953.959611:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1787828953.959612:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828953.959613:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828953.959637:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787828953.959655:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828953.959658:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298426916 00000100:00000001:0.0:1787828953.959660:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298428160 00000100:00000001:0.0:1787828953.959661:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298426916 00000100:00000001:0.0:1787828953.959663:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298428160 00000100:00000001:0.0:1787828953.959664:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787828953.959666:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828953.959667:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787828953.959670:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828953/real 1787828953] req@ffff8d30d13ee580 x1874670646781696/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828978 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787828953.959676:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787828953.959677:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787828953.959678:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787828953.959679:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787828953.959680:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828953.959682:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828953.959683:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787828953.959686:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d13ee580 x1874670646781696/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828978 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787828953.959691:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828953.959692:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787828953.959694:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828953.959695:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828953.959696:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828953.959697:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787828953.959698:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787828953.959700:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828953.959702:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787828953.959703:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787828953.959721:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d13ee580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646781696:192.168.203.143@tcp:8:kworker.0 00000100:00000001:0.0:1787828953.959724:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828953.959725:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828953.959726:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787828953.959727:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787828953.959729:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787828953.959730:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787828953.959731:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787828953.959733:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787828953.959734:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787828953.959736:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787828953.959737:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828953.959738:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828953.959739:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787828953.959740:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787828953.959741:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787828953.959741:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787828953.959742:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787828953.959743:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787828953.959744:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787828953.959745:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787828953.959746:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828953.959747:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828953.959749:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787828953.959751:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828953.959752:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787828953.959753:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828954.854716:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828954.854722:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828954.854803:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828954.854807:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828954.854810:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828955.878716:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828955.878722:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828955.878783:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828955.878787:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828955.878790:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828956.902733:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828956.902740:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828956.902817:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828956.902821:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828956.902825:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828957.926701:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828957.926706:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828957.926776:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828957.926782:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828957.926787:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828958.950681:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828958.950687:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828958.950735:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828958.950738:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828958.950741:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787828959.078689:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828959.078696:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1787828959.078699:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787828959.078702:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787828959.078705:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787828959.078711:0:40701:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 3759 00000100:00080000:2.0:1787828959.078720:0:40701:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787828959.078722:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828959.078724:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787828959.078727:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787828959.078730:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787828959.078733:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:2.0:1787828959.078736:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:2.0:1787828959.078738:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828959.078740:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:2.0:1787828959.078744:0:40701:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787828959.078747:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828959.078749:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787828959.078753:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828959.078756:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828959.078759:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:2.0:1787828959.078761:0:40701:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828959.078773:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787828959.078775:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828959.078778:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787828959.078781:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787828959.078785:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828959.078794:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787828959.078802:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2c780 x1874670646781824/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787828959.078810:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787828959.078813:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828959.078826:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787828959.078833:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828959.078835:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1787828959.078837:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787828959.078838:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787828959.078840:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787828959.078842:0:40701:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3759 00000100:00080000:2.0:1787828959.078845:0:40701:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787828959.078846:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828959.078847:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787828959.078849:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787828959.078850:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787828959.078852:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000020:00000001:2.0:1787828959.078854:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:2.0:1787828959.078856:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828959.078857:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:2.0:1787828959.078859:0:40701:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787828959.078861:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828959.078862:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:0.0:1787828959.078864:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:2.0:1787828959.078867:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787828959.078870:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 02000000:00000001:2.0:1787828959.078871:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828959.078874:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1787828959.078876:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000008:00100000:2.0:1787828959.078878:0:40701:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00000001:0.0:1787828959.078880:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000008:00000001:2.0:1787828959.078881:0:40701:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828959.078885:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828959.078886:0:40701:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828959.078891:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787828959.078894:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787828959.078894:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828959.078898:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787828959.078900:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828959.078900:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787828959.078904:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828959.078906:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828959.078908:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828959.078915:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646781824:192.168.203.143@tcp:250:kworker.0 00000100:00100000:2.0:1787828959.078916:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2fc00 x1874670646781952/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787828959.078926:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:2.0:1787828959.078928:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787828959.078931:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828959.078932:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:2.0:1787828959.078933:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828959.078937:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787828959.078938:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 02000000:00000001:0.0:1787828959.078944:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00080000:2.0:1787828959.078945:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1787828959.078948:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787828959.078949:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787828959.078951:0:40701:0:(import.c:485:import_select_connection()) Process entered 02000000:00000001:0.0:1787828959.078953:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787828959.078954:0:40701:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3759 00000100:00080000:2.0:1787828959.078958:0:40701:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787828959.078960:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828959.078961:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787828959.078963:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787828959.078963:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787828959.078965:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000020:00000001:2.0:1787828959.078966:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:2.0:1787828959.078968:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828959.078970:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:0.0:1787828959.078971:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00080000:2.0:1787828959.078973:0:40701:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787828959.078975:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417870208 : -126234291681408 : ffff8d30c9e2c780) 00000100:00000001:2.0:1787828959.078976:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828959.078981:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:0.0:1787828959.078982:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 02000000:00000001:2.0:1787828959.078983:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828959.078987:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828959.078988:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1787828959.078990:0:40701:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1787828959.078992:0:40701:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828959.078993:0:40701:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828959.078997:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787828959.078998:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828959.078999:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787828959.079000:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787828959.079001:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828959.079003:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787828959.079006:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2c3c0 x1874670646782080/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787828959.079012:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787828959.079013:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828959.079015:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787828959.079017:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828959.079019:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1787828959.079020:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787828959.079021:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787828959.079023:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787828959.079025:0:40701:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3759 00000100:00080000:2.0:1787828959.079027:0:40701:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787828959.079029:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828959.079030:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787828959.079031:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:0.0:1787828959.079032:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1787828959.079033:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787828959.079036:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:0.0:1787828959.079040:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4298433280 00000020:00000001:2.0:1787828959.079040:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:2.0:1787828959.079044:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828959.079046:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:2.0:1787828959.079049:0:40701:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787828959.079051:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828959.079052:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 02000000:00000001:2.0:1787828959.079053:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787828959.079056:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787828959.079062:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4298433280 02000000:00000001:2.0:1787828959.079067:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828959.079069:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1787828959.079071:0:40701:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00000001:0.0:1787828959.079072:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 00000008:00000001:2.0:1787828959.079073:0:40701:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828959.079077:0:40701:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828959.079080:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828959.079082:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787828959.079085:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828959.079086:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828959.079087:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787828959.079090:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:0.0:1787828959.079093:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828959.079093:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828959.079098:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828959.079099:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787828959.079107:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2de00 x1874670646782208/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787828959.079110:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787828959.079114:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:2.0:1787828959.079115:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787828959.079118:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828959.079120:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787828959.079121:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828959.079125:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787828959.079129:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1787828959.079131:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787828959.079133:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828959.079136:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787828959.079139:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787828959.079142:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828959.079147:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646781952:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787828959.079152:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787828959.079154:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787828959.079156:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828959.079158:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1787828959.079162:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828959.079170:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787828959.079171:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417883648 : -126234291667968 : ffff8d30c9e2fc00) 00000100:00000001:0.0:1787828959.079176:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:0.0:1787828959.079189:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828959.079194:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298432022 00000100:00000001:0.0:1787828959.079198:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298433280 00000100:00000001:0.0:1787828959.079202:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298432022 00000100:00000001:0.0:1787828959.079204:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298433280 00000100:00000001:0.0:1787828959.079206:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787828959.079209:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828959.079211:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1787828959.079212:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787828959.079214:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828959.079216:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787828959.079218:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787828959.079221:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828959.079225:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646782080:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787828959.079229:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787828959.079231:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787828959.079232:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828959.079234:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1787828959.079237:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828959.079242:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787828959.079244:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417869248 : -126234291682368 : ffff8d30c9e2c3c0) 00000100:00000001:0.0:1787828959.079248:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:0.0:1787828959.079257:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828959.079260:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298432022 00000100:00000001:0.0:1787828959.079263:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298433280 00000100:00000001:0.0:1787828959.079265:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298432022 00000100:00000001:0.0:1787828959.079268:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298433280 00000100:00000001:0.0:1787828959.079270:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787828959.079272:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828959.079274:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1787828959.079276:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787828959.079277:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828959.079280:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787828959.079282:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787828959.079284:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828959.079288:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646782208:192.168.203.143@tcp:8:kworker.0 00000100:00000001:0.0:1787828959.079291:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787828959.079293:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787828959.079295:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828959.079297:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1787828959.079299:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828959.079304:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787828959.079306:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417875968 : -126234291675648 : ffff8d30c9e2de00) 00000100:00000001:0.0:1787828959.079309:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:0.0:1787828959.079317:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828959.079321:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298432035 00000100:00000001:0.0:1787828959.079324:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298433280 00000100:00000001:0.0:1787828959.079326:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298432035 00000100:00000001:0.0:1787828959.079329:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298433280 00000100:00000001:0.0:1787828959.079331:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787828959.079334:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828959.079336:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828959.079339:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828959.079342:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787828959.079344:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:0.0:1787828959.079346:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787828959.079348:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787828959.079350:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828959.079352:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828959.974714:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828959.974721:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828959.974783:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828959.974788:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828959.974792:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828960.998719:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828960.998725:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828960.998800:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828960.998805:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828960.998810:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828962.022701:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828962.022706:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828962.022789:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828962.022794:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828962.022797:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:3.0:1787828962.150952:0:2381:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.143:988 00000400:00000100:3.0:1787828962.151028:0:2381:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.143@tcp at host 192.168.203.143 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:3.0:1787828962.151050:0:2381:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.143@tcp 00000100:00000001:3.0:1787828962.151069:0:2381:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787828962.151086:0:2381:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787828962.151089:0:2381:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828962.151091:0:2381:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:3.0:1787828962.151107:0:2381:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.143@tcp 00000100:00000001:0.0:1787828962.151113:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1787828962.151114:0:2381:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1787828962.151119:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787828962.151119:0:2381:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787828962.151121:0:2381:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828962.151123:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00000001:3.0:1787828962.151125:0:2381:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:3.0:1787828962.151133:0:2381:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.143@tcp 00000100:00100000:0.0:1787828962.151138:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828959/real 1787828962] req@ffff8d30c9e2fc00 x1874670646781952/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828984 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787828962.151140:0:2381:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787828962.151145:0:2381:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787828962.151148:0:2381:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828962.151152:0:2381:0:(events.c:68:request_out_callback()) Process leaving 00000400:00000001:0.0:1787828962.151155:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787828962.151159:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787828962.151165:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000800:00000100:3.0:1787828962.151167:0:2381:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.143@tcp 00000100:00000001:0.0:1787828962.151169:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787828962.151174:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787828962.151177:0:2381:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787828962.151182:0:2381:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828962.151184:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787828962.151185:0:2381:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828962.151189:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787828962.151190:0:2381:0:(events.c:68:request_out_callback()) Process leaving 00000100:00100000:0.0:1787828962.151200:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2fc00 x1874670646781952/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828984 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787828962.151210:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828962.151214:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787828962.151218:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828962.151220:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828962.151221:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828962.151225:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787828962.151228:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787828962.151231:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828962.151238:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787828962.151241:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787828962.151247:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646781952:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787828962.151253:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787828962.151258:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828959/real 1787828962] req@ffff8d30c9e2c3c0 x1874670646782080/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828984 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787828962.151268:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787828962.151269:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787828962.151272:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787828962.151274:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787828962.151276:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828962.151278:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828962.151279:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787828962.151283:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2c3c0 x1874670646782080/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828984 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787828962.151290:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828962.151292:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787828962.151295:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828962.151297:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828962.151298:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828962.151300:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787828962.151302:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787828962.151304:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828962.151308:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787828962.151311:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787828962.151314:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646782080:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787828962.151318:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787828962.151322:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828959/real 1787828962] req@ffff8d30c9e2de00 x1874670646782208/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828984 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787828962.151330:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787828962.151332:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787828962.151335:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787828962.151337:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787828962.151338:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828962.151340:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828962.151341:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787828962.151345:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2de00 x1874670646782208/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828984 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787828962.151352:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828962.151353:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787828962.151356:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828962.151357:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828962.151358:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828962.151374:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787828962.151377:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787828962.151380:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828962.151384:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787828962.151386:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787828962.151389:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646782208:192.168.203.143@tcp:8:kworker.0 00000100:00000001:0.0:1787828962.151394:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787828962.151399:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828959/real 1787828962] req@ffff8d30c9e2c780 x1874670646781824/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828984 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787828962.151417:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787828962.151419:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787828962.151423:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787828962.151425:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787828962.151426:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828962.151428:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828962.151430:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787828962.151435:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2c780 x1874670646781824/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828984 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787828962.151452:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828962.151454:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787828962.151457:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828962.151458:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828962.151460:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828962.151462:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787828962.151464:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787828962.151466:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828962.151470:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787828962.151472:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787828962.151476:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646781824:192.168.203.143@tcp:250:kworker.0 00000100:00000001:0.0:1787828962.151481:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828962.151484:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828962.151486:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787828962.151490:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787828962.151496:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787828962.151500:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787828962.151503:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787828962.151508:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787828962.151511:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787828962.151515:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787828962.151516:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828962.151519:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828962.151520:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787828962.151521:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787828962.151524:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787828962.151525:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787828962.151526:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787828962.151528:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787828962.151529:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787828962.151532:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787828962.151533:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828962.151535:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828962.151536:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787828962.151538:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787828962.151540:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787828962.151542:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787828962.151543:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787828962.151545:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787828962.151546:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787828962.151552:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787828962.151553:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828962.151555:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828962.151562:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787828962.151563:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787828962.151565:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787828962.151567:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787828962.151568:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787828962.151570:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787828962.151571:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787828962.151573:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787828962.151574:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828962.151577:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828962.151587:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787828962.151590:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828962.151592:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787828962.151594:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828963.046736:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828963.046746:0:815:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1787828963.046749:0:815:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1787828963.046753:0:815:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828963.046758:0:815:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1787828963.046761:0:815:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828963.046762:0:815:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828963.046764:0:815:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828963.046851:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828963.046857:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828963.046862:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828964.070681:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828964.070688:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828964.070746:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828964.070752:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828964.070756:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787828964.198834:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828964.198846:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1787828964.198850:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787828964.198854:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787828964.198860:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787828964.198868:0:40701:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 3764 00000100:00080000:2.0:1787828964.198881:0:40701:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787828964.198885:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828964.198887:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787828964.198898:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787828964.198901:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787828964.198905:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:2.0:1787828964.198907:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:2.0:1787828964.198909:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828964.198911:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:2.0:1787828964.198915:0:40701:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787828964.198917:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828964.198920:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787828964.198923:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828964.198926:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828964.198929:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:2.0:1787828964.198949:0:40701:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828964.198961:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787828964.198963:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828964.198968:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787828964.198972:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787828964.198979:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828964.198995:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787828964.199025:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2e940 x1874670646782336/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787828964.199038:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787828964.199044:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828964.199071:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787828964.199075:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828964.199078:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1787828964.199080:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787828964.199082:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787828964.199086:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787828964.199089:0:40701:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3764 00000100:00080000:2.0:1787828964.199094:0:40701:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787828964.199097:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828964.199098:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787828964.199101:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787828964.199103:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787828964.199105:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000020:00000001:2.0:1787828964.199108:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:2.0:1787828964.199110:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828964.199112:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:2.0:1787828964.199115:0:40701:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787828964.199118:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828964.199120:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787828964.199127:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828964.199129:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828964.199132:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1787828964.199138:0:40701:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1787828964.199141:0:40701:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828964.199144:0:40701:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828964.199149:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787828964.199151:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828964.199153:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787828964.199154:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787828964.199157:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828964.199163:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787828964.199170:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2f0c0 x1874670646782464/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787828964.199178:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787828964.199181:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828964.199184:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787828964.199186:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828964.199191:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1787828964.199193:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787828964.199194:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787828964.199197:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787828964.199200:0:40701:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3764 00000100:00080000:2.0:1787828964.199205:0:40701:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787828964.199207:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828964.199208:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787828964.199211:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787828964.199212:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787828964.199215:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000020:00000001:2.0:1787828964.199217:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:2.0:1787828964.199219:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828964.199220:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:2.0:1787828964.199230:0:40701:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787828964.199233:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828964.199236:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787828964.199239:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828964.199243:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828964.199245:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1787828964.199248:0:40701:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1787828964.199251:0:40701:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828964.199253:0:40701:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828964.199256:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787828964.199258:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828964.199260:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787828964.199261:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787828964.199263:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828964.199268:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787828964.199273:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2ed00 x1874670646782592/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787828964.199281:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787828964.199283:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828964.199285:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787828964.199288:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828964.199292:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1787828964.199294:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787828964.199296:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787828964.199299:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787828964.199302:0:40701:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3764 00000100:00080000:2.0:1787828964.199306:0:40701:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787828964.199308:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828964.199310:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787828964.199314:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787828964.199315:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787828964.199317:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000020:00000001:2.0:1787828964.199320:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:2.0:1787828964.199323:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828964.199325:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:2.0:1787828964.199328:0:40701:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787828964.199332:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828964.199334:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787828964.199338:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828964.199341:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828964.199344:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1787828964.199365:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000008:00100000:2.0:1787828964.199365:0:40701:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000008:00000001:2.0:1787828964.199373:0:40701:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828964.199374:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1787828964.199377:0:40701:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828964.199381:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1787828964.199386:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 02000000:00000001:2.0:1787828964.199387:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:0.0:1787828964.199389:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828964.199392:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828964.199397:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787828964.199398:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787828964.199400:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787828964.199405:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828964.199406:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828964.199413:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828964.199420:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828964.199424:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646782336:192.168.203.143@tcp:250:kworker.0 00000100:00100000:2.0:1787828964.199434:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2c000 x1874670646782720/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787828964.199435:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787828964.199444:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787828964.199450:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828964.199450:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787828964.199457:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828964.199458:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:2.0:1787828964.199467:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828964.199468:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828964.199495:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787828964.199498:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417878848 : -126234291672768 : ffff8d30c9e2e940) 00000100:00000001:0.0:1787828964.199504:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1787828964.199561:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:0.0:1787828964.199578:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1787828964.199583:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828964.199586:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828964.199588:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787828964.199597:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828964.199646:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4298438401 00000100:00000001:0.0:1787828964.199661:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1787828964.199668:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4298438401 00000100:00000001:0.0:1787828964.199673:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1787828964.199677:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787828964.199681:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828964.199684:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787828964.199690:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828964/real 1787828964] req@ffff8d30c9e2e940 x1874670646782336/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828989 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787828964.199714:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787828964.199717:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787828964.199721:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787828964.199724:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787828964.199726:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828964.199729:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828964.199732:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787828964.199737:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2e940 x1874670646782336/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828989 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787828964.199745:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828964.199749:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787828964.199753:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828964.199755:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828964.199756:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828964.199760:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787828964.199762:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787828964.199765:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828964.199771:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787828964.199775:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787828964.199780:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646782336:192.168.203.143@tcp:250:kworker.0 00000100:00000001:0.0:1787828964.199786:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1787828964.199788:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787828964.199790:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828964.199798:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787828964.199800:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787828964.199803:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828964.199806:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646782464:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787828964.199811:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787828964.199814:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787828964.199815:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828964.199818:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1787828964.199821:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828964.199828:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787828964.199830:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417880768 : -126234291670848 : ffff8d30c9e2f0c0) 00000100:00000001:0.0:1787828964.199836:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1787828964.199852:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:0.0:1787828964.199858:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1787828964.199861:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828964.199862:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828964.199864:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787828964.199867:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828964.199872:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298437143 00000100:00000001:0.0:1787828964.199875:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298438401 00000100:00000001:0.0:1787828964.199878:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298437143 00000100:00000001:0.0:1787828964.199880:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298438401 00000100:00000001:0.0:1787828964.199883:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787828964.199885:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828964.199887:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787828964.199892:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828964/real 1787828964] req@ffff8d30c9e2f0c0 x1874670646782464/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828989 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787828964.199908:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787828964.199909:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787828964.199912:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787828964.199914:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787828964.199916:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828964.199917:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828964.199919:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787828964.199923:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2f0c0 x1874670646782464/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828989 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787828964.199930:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828964.199932:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787828964.199934:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828964.199936:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828964.199937:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828964.199939:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787828964.199941:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787828964.199944:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828964.199947:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787828964.199950:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787828964.199953:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646782464:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787828964.199957:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1787828964.199959:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787828964.199960:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828964.199963:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787828964.199964:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787828964.199967:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828964.199970:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646782592:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787828964.199974:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787828964.199976:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787828964.199978:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828964.199980:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1787828964.199997:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828964.200004:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787828964.200005:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417879808 : -126234291671808 : ffff8d30c9e2ed00) 00000100:00000001:0.0:1787828964.200010:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1787828964.200023:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:0.0:1787828964.200027:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1787828964.200030:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828964.200032:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828964.200034:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787828964.200037:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828964.200041:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298437143 00000100:00000001:0.0:1787828964.200045:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298438401 00000100:00000001:0.0:1787828964.200047:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298437143 00000100:00000001:0.0:1787828964.200051:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298438401 00000100:00000001:0.0:1787828964.200053:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787828964.200055:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828964.200057:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787828964.200062:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828964/real 1787828964] req@ffff8d30c9e2ed00 x1874670646782592/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828989 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787828964.200073:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787828964.200074:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787828964.200078:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787828964.200081:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787828964.200083:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828964.200086:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828964.200088:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787828964.200093:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2ed00 x1874670646782592/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828989 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787828964.200101:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828964.200104:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787828964.200108:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828964.200111:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828964.200113:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828964.200114:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787828964.200116:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787828964.200120:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828964.200125:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787828964.200128:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787828964.200132:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646782592:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787828964.200137:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828964.200139:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828964.200141:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787828964.200144:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787828964.200148:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787828964.200151:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787828964.200153:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787828964.200158:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787828964.200160:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787828964.200165:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787828964.200166:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828964.200168:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828964.200170:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787828964.200171:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787828964.200173:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787828964.200174:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787828964.200175:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787828964.200178:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787828964.200179:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787828964.200183:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787828964.200184:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828964.200186:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828964.200187:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787828964.200188:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787828964.200191:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787828964.200192:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787828964.200194:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787828964.200195:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787828964.200197:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787828964.200199:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787828964.200201:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828964.200204:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828964.200216:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787828964.200220:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828964.200223:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787828964.200225:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787828964.200227:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1787828964.200228:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787828964.200230:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828964.200233:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787828964.200235:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787828964.200238:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828964.200242:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646782720:192.168.203.143@tcp:8:kworker.0 00000100:00000001:0.0:1787828964.200245:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787828964.200247:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787828964.200248:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828964.200250:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1787828964.200254:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828964.200262:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787828964.200263:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417868288 : -126234291683328 : ffff8d30c9e2c000) 00000100:00000001:0.0:1787828964.200267:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1787828964.200282:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:0.0:1787828964.200287:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1787828964.200290:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828964.200292:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828964.200294:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787828964.200297:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828964.200300:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298437156 00000100:00000001:0.0:1787828964.200303:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298438401 00000100:00000001:0.0:1787828964.200306:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298437156 00000100:00000001:0.0:1787828964.200308:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298438401 00000100:00000001:0.0:1787828964.200310:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787828964.200312:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828964.200316:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787828964.200320:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828964/real 1787828964] req@ffff8d30c9e2c000 x1874670646782720/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828989 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787828964.200329:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787828964.200331:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787828964.200334:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787828964.200336:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787828964.200338:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828964.200340:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828964.200342:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787828964.200360:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2c000 x1874670646782720/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828989 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787828964.200367:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828964.200369:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787828964.200371:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828964.200373:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828964.200374:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828964.200375:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787828964.200377:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787828964.200379:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828964.200381:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787828964.200383:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787828964.200385:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646782720:192.168.203.143@tcp:8:kworker.0 00000100:00000001:0.0:1787828964.200387:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828964.200389:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828964.200390:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787828964.200391:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787828964.200393:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787828964.200394:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787828964.200396:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787828964.200397:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787828964.200398:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787828964.200402:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787828964.200403:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828964.200404:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828964.200407:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787828964.200408:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828964.200409:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787828964.200411:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828965.094701:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828965.094709:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828965.094781:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828965.094785:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828965.094788:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828966.118772:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828966.118780:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828966.118857:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828966.118862:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828966.118866:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828967.142689:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828967.142697:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828967.142749:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828967.142753:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828967.142756:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828968.166757:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828968.166764:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828968.166849:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828968.166854:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828968.166999:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828969.190739:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828969.190746:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828969.190794:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828969.190800:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828969.190803:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787828969.318765:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828969.318776:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1787828969.318779:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1787828969.318785:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787828969.318792:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1787828969.318799:0:40701:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 3769 00000100:00080000:1.0:1787828969.318813:0:40701:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1787828969.318818:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787828969.318821:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:1.0:1787828969.318826:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:1.0:1787828969.318830:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:1.0:1787828969.318834:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:1.0:1787828969.318838:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:1.0:1787828969.318841:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787828969.318844:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:1.0:1787828969.318849:0:40701:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:1.0:1787828969.318853:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828969.318856:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1787828969.318860:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787828969.318864:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828969.318867:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:1.0:1787828969.318869:0:40701:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828969.318879:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1787828969.318882:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828969.318887:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1787828969.318889:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1787828969.318893:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828969.318905:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787828969.318916:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d13ef480 x1874670646782848/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787828969.318927:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787828969.318932:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828969.318956:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1787828969.318961:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828969.318965:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1787828969.318968:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1787828969.318970:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787828969.318975:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1787828969.318980:0:40701:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3769 00000100:00080000:1.0:1787828969.318986:0:40701:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787828969.318988:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787828969.318990:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787828969.318994:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:2.0:1787828969.319000:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000020:00000001:1.0:1787828969.319001:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:1.0:1787828969.319007:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:1.0:1787828969.319012:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000020:00000001:1.0:1787828969.319017:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:1.0:1787828969.319020:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787828969.319022:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:1.0:1787828969.319027:0:40701:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:1.0:1787828969.319030:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828969.319034:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:2.0:1787828969.319035:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1787828969.319038:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 02000000:00000001:1.0:1787828969.319041:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787828969.319041:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828969.319051:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787828969.319052:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828969.319056:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787828969.319058:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 02000000:00000001:2.0:1787828969.319061:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000008:00100000:1.0:1787828969.319067:0:40701:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787828969.319071:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d13ef480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646782848:192.168.203.143@tcp:250:kworker.0 00000008:00000001:1.0:1787828969.319075:0:40701:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828969.319078:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:1.0:1787828969.319084:0:40701:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828969.319087:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1787828969.319091:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828969.319093:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787828969.319094:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787828969.319099:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828969.319101:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828969.319104:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1787828969.319109:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1787828969.319113:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828969.319121:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828969.319122:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787828969.319125:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839541351552 : -126234168200064 : ffff8d30d13ef480) 00000100:00000001:2.0:1787828969.319130:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00100000:1.0:1787828969.319131:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d13ef840 x1874670646782976/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787828969.319146:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787828969.319148:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828969.319152:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1787828969.319155:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828969.319159:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:1.0:1787828969.319161:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1787828969.319163:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787828969.319165:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1787828969.319167:0:40701:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3769 00000100:00080000:1.0:1787828969.319171:0:40701:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1787828969.319173:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787828969.319174:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000800:00000100:2.0:1787828969.319180:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000020:00000001:1.0:1787828969.319182:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:1.0:1787828969.319185:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:1.0:1787828969.319187:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000020:00000001:1.0:1787828969.319191:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:1.0:1787828969.319193:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787828969.319195:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:2.0:1787828969.319196:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00080000:1.0:1787828969.319200:0:40701:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787828969.319202:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787828969.319203:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828969.319206:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:2.0:1787828969.319207:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828969.319211:0:2390:0:(events.c:68:request_out_callback()) Process leaving 02000000:00000001:1.0:1787828969.319212:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787828969.319217:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828969.319218:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828969.319219:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:1.0:1787828969.319226:0:40701:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00000001:2.0:1787828969.319227:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4298443520 00000008:00000001:1.0:1787828969.319236:0:40701:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828969.319237:0:40701:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828969.319241:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:2.0:1787828969.319242:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 02000000:00000001:1.0:1787828969.319243:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828969.319251:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1787828969.319253:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1787828969.319254:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828969.319257:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4298443520 00000100:00000001:1.0:1787828969.319261:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828969.319265:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 00000100:00100000:1.0:1787828969.319270:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d13eed00 x1874670646783104/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787828969.319271:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787828969.319280:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828969.319291:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00000001:1.0:1787828969.319298:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787828969.319302:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787828969.319304:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828969/real 1787828969] req@ffff8d30d13ef480 x1874670646782848/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828994 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787828969.319306:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1787828969.319327:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828969.319337:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000400:00000001:2.0:1787828969.319337:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000001:1.0:1787828969.319342:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000400:00000001:2.0:1787828969.319345:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00080000:1.0:1787828969.319346:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787828969.319351:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:2.0:1787828969.319353:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00080000:1.0:1787828969.319357:0:40701:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3769 00000100:00000001:2.0:1787828969.319361:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00080000:1.0:1787828969.319363:0:40701:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787828969.319364:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828969.319370:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828969.319370:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787828969.319373:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:2.0:1787828969.319375:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:1.0:1787828969.319377:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:1.0:1787828969.319381:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:1.0:1787828969.319384:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00100000:2.0:1787828969.319385:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d13ef480 x1874670646782848/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828994 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:1.0:1787828969.319392:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:1.0:1787828969.319404:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787828969.319411:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:2.0:1787828969.319416:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828969.319421:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787828969.319422:0:40701:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787828969.319426:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828969.319433:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787828969.319434:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828969.319435:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:1.0:1787828969.319450:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:2.0:1787828969.319450:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787828969.319454:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 02000000:00000001:1.0:1787828969.319457:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00080000:2.0:1787828969.319459:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 02000000:00000001:1.0:1787828969.319463:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828969.319466:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00080000:2.0:1787828969.319467:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000008:00100000:1.0:1787828969.319472:0:40701:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00000001:2.0:1787828969.319474:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000008:00000001:1.0:1787828969.319476:0:40701:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828969.319480:0:40701:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787828969.319481:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d13ef480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646782848:192.168.203.143@tcp:250:kworker.0 02000000:00000001:1.0:1787828969.319491:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:2.0:1787828969.319494:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:1.0:1787828969.319495:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828969.319507:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:2.0:1787828969.319508:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 02000000:00000001:1.0:1787828969.319519:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:2.0:1787828969.319520:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787828969.319529:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787828969.319530:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828969.319537:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00000001:1.0:1787828969.319539:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1787828969.319540:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:2.0:1787828969.319544:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 00000100:00100000:1.0:1787828969.319550:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d13ede00 x1874670646783232/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:2.0:1787828969.319551:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787828969.319555:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1787828969.319560:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787828969.319562:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787828969.319563:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828969.319568:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828969.319569:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828969.319575:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828969.319581:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1787828969.319583:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828969.319585:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787828969.319587:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1787828969.319589:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1787828969.319591:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787828969.319592:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828969.319594:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787828969.319597:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828969.319629:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787828969.319634:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d13ef840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646782976:192.168.203.143@tcp:38:kworker.0 00000100:00000001:2.0:1787828969.319637:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787828969.319640:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1787828969.319642:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828969.319644:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1787828969.319647:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828969.319655:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787828969.319656:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839541352512 : -126234168199104 : ffff8d30d13ef840) 00000100:00000001:2.0:1787828969.319661:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1787828969.319672:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:2.0:1787828969.319676:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1787828969.319678:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828969.319679:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828969.319680:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1787828969.319682:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828969.319686:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298442263 00000100:00000001:2.0:1787828969.319688:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298443520 00000100:00000001:2.0:1787828969.319690:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298442263 00000100:00000001:2.0:1787828969.319691:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298443520 00000100:00000001:2.0:1787828969.319693:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787828969.319694:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828969.319695:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1787828969.319698:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828969/real 1787828969] req@ffff8d30d13ef840 x1874670646782976/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828994 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1787828969.319703:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787828969.319704:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1787828969.319706:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1787828969.319707:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787828969.319708:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828969.319709:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828969.319711:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787828969.319713:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d13ef840 x1874670646782976/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828994 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787828969.319722:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828969.319723:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787828969.319725:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828969.319726:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828969.319727:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828969.319728:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787828969.319729:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787828969.319731:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828969.319733:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:2.0:1787828969.319734:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1787828969.319736:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d13ef840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646782976:192.168.203.143@tcp:38:kworker.0 00000100:00000001:2.0:1787828969.319738:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1787828969.319739:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787828969.319740:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828969.319742:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787828969.319743:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828969.319745:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787828969.319747:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d13eed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646783104:192.168.203.143@tcp:38:kworker.0 00000100:00000001:2.0:1787828969.319749:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787828969.319750:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1787828969.319751:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828969.319752:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1787828969.319755:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828969.319758:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787828969.319759:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839541349632 : -126234168201984 : ffff8d30d13eed00) 00000100:00000001:2.0:1787828969.319762:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1787828969.319768:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:2.0:1787828969.319770:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1787828969.319772:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828969.319773:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828969.319775:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1787828969.319776:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828969.319778:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298442263 00000100:00000001:2.0:1787828969.319780:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298443520 00000100:00000001:2.0:1787828969.319782:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298442263 00000100:00000001:2.0:1787828969.319783:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298443520 00000100:00000001:2.0:1787828969.319785:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787828969.319786:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828969.319787:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1787828969.319789:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828969/real 1787828969] req@ffff8d30d13eed00 x1874670646783104/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828994 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1787828969.319793:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787828969.319794:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1787828969.319795:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1787828969.319796:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787828969.319797:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828969.319798:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828969.319799:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787828969.319801:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d13eed00 x1874670646783104/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828994 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787828969.319806:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828969.319807:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787828969.319809:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828969.319810:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828969.319811:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828969.319812:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787828969.319813:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787828969.319814:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828969.319816:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:2.0:1787828969.319817:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1787828969.319820:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d13eed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646783104:192.168.203.143@tcp:38:kworker.0 00000100:00000001:2.0:1787828969.319823:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1787828969.319824:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787828969.319825:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828969.319826:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787828969.319827:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828969.319828:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787828969.319830:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d13ede00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646783232:192.168.203.143@tcp:8:kworker.0 00000100:00000001:2.0:1787828969.319832:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787828969.319833:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1787828969.319834:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828969.319835:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1787828969.319837:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828969.319840:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787828969.319841:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839541345792 : -126234168205824 : ffff8d30d13ede00) 00000100:00000001:2.0:1787828969.319843:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1787828969.319848:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:2.0:1787828969.319850:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1787828969.319851:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828969.319852:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828969.319854:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1787828969.319855:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828969.319857:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298442276 00000100:00000001:2.0:1787828969.319859:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298443520 00000100:00000001:2.0:1787828969.319860:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298442276 00000100:00000001:2.0:1787828969.319862:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298443520 00000100:00000001:2.0:1787828969.319863:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787828969.319864:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828969.319865:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1787828969.319867:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828969/real 1787828969] req@ffff8d30d13ede00 x1874670646783232/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828994 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1787828969.319876:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787828969.319877:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1787828969.319878:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1787828969.319879:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787828969.319880:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828969.319881:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828969.319882:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787828969.319884:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d13ede00 x1874670646783232/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828994 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787828969.319888:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828969.319889:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787828969.319891:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828969.319892:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828969.319892:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828969.319893:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787828969.319894:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787828969.319896:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787828969.319898:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:2.0:1787828969.319899:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1787828969.319901:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d13ede00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646783232:192.168.203.143@tcp:8:kworker.0 00000100:00000001:2.0:1787828969.319903:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828969.319904:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828969.319905:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787828969.319906:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1787828969.319907:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787828969.319908:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:2.0:1787828969.319909:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:2.0:1787828969.319910:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787828969.319911:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1787828969.319913:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787828969.319914:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828969.319915:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828969.319916:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787828969.319917:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1787828969.319918:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787828969.319919:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:2.0:1787828969.319920:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:2.0:1787828969.319921:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787828969.319922:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1787828969.319924:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787828969.319924:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828969.319925:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787828969.319926:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787828969.319927:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1787828969.319928:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787828969.319928:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:2.0:1787828969.319929:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:2.0:1787828969.319930:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787828969.319930:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1787828969.319932:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787828969.319933:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828969.319934:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828969.319936:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1787828969.319936:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828969.319938:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787828969.319939:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828970.214745:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828970.214752:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828970.214834:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828970.214838:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828970.214841:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828971.238740:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828971.238760:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828971.238846:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828971.238853:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828971.238858:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828972.262695:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828972.262699:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828972.262834:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828972.262839:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828972.262842:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828973.286740:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828973.286749:0:815:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1787828973.286757:0:815:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1787828973.286761:0:815:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828973.286766:0:815:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1787828973.286769:0:815:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828973.286771:0:815:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828973.286790:0:815:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828973.286847:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828973.286851:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828973.286854:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828974.310693:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828974.310698:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828974.310758:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828974.310762:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828974.310765:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787828974.438691:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828974.438702:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1787828974.438704:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787828974.438708:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787828974.438713:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787828974.438719:0:40701:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 3774 00000100:00080000:2.0:1787828974.438734:0:40701:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787828974.438737:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828974.438739:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787828974.438744:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787828974.438746:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787828974.438749:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:2.0:1787828974.438752:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:2.0:1787828974.438754:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828974.438756:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:2.0:1787828974.438760:0:40701:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787828974.438763:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828974.438765:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787828974.438769:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828974.438772:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828974.438774:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:2.0:1787828974.438777:0:40701:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828974.438787:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787828974.438789:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828974.438793:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787828974.438795:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787828974.438800:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828974.438811:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787828974.438820:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2da40 x1874670646783360/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787828974.438830:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787828974.438835:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828974.438854:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787828974.438857:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828974.438861:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1787828974.438862:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787828974.438864:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787828974.438867:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787828974.438869:0:40701:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3774 00000100:00080000:2.0:1787828974.438874:0:40701:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787828974.438877:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828974.438879:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787828974.438882:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787828974.438884:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787828974.438886:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:0.0:1787828974.438888:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:2.0:1787828974.438889:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:0.0:1787828974.438894:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1787828974.438894:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828974.438896:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:0.0:1787828974.438898:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1787828974.438902:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00080000:2.0:1787828974.438906:0:40701:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787828974.438909:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828974.438911:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:0.0:1787828974.438913:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828974.438917:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787828974.438920:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787828974.438922:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828974.438925:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787828974.438926:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 02000000:00000001:0.0:1787828974.438932:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000008:00100000:2.0:1787828974.438933:0:40701:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1787828974.438939:0:40701:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828974.438941:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2da40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646783360:192.168.203.143@tcp:250:kworker.0 00000100:00000001:2.0:1787828974.438943:0:40701:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828974.438951:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787828974.438952:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787828974.438955:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828974.438956:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:2.0:1787828974.438958:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787828974.438960:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828974.438962:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787828974.438964:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1787828974.438967:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828974.438970:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828974.438975:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787828974.438984:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2f480 x1874670646783488/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787828974.438985:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787828974.438989:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417875008 : -126234291676608 : ffff8d30c9e2da40) 00000100:00000001:0.0:1787828974.438995:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:2.0:1787828974.439000:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787828974.439003:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828974.439007:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787828974.439010:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828974.439014:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1787828974.439015:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787828974.439017:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787828974.439020:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787828974.439024:0:40701:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3774 00000100:00080000:2.0:1787828974.439029:0:40701:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787828974.439031:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828974.439033:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000800:00000100:0.0:1787828974.439038:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000020:00000001:2.0:1787828974.439038:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787828974.439042:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787828974.439045:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000020:00000001:2.0:1787828974.439048:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:2.0:1787828974.439050:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787828974.439052:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1787828974.439053:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:0.0:1787828974.439058:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00080000:2.0:1787828974.439061:0:40701:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787828974.439063:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828974.439066:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1787828974.439068:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828974.439072:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:0.0:1787828974.439073:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828974.439076:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787828974.439081:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4298448640 02000000:00000001:2.0:1787828974.439081:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828974.439084:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1787828974.439088:0:40701:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1787828974.439091:0:40701:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828974.439093:0:40701:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828974.439094:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 02000000:00000001:2.0:1787828974.439099:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:0.0:1787828974.439103:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4298448640 02000000:00000001:2.0:1787828974.439103:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828974.439107:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787828974.439111:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:0.0:1787828974.439112:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 02000000:00000001:2.0:1787828974.439113:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828974.439119:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787828974.439123:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828974.439123:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828974.439130:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1787828974.439135:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2c000 x1874670646783616/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787828974.439140:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828974/real 1787828974] req@ffff8d30c9e2da40 x1874670646783360/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828999 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787828974.439153:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787828974.439158:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828974.439162:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:0.0:1787828974.439163:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00080000:2.0:1787828974.439166:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000001:0.0:1787828974.439167:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00080000:2.0:1787828974.439173:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1787828974.439174:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1787828974.439176:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:0.0:1787828974.439179:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00080000:2.0:1787828974.439180:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787828974.439183:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828974.439185:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:0.0:1787828974.439186:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828974.439190:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:2.0:1787828974.439193:0:40701:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3774 00000100:00100000:0.0:1787828974.439197:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2da40 x1874670646783360/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787828999 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787828974.439201:0:40701:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787828974.439207:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828974.439207:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787828974.439212:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787828974.439212:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:0.0:1787828974.439218:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:2.0:1787828974.439219:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:0.0:1787828974.439221:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000020:00000001:2.0:1787828974.439223:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:0.0:1787828974.439224:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828974.439229:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000020:00000001:2.0:1787828974.439229:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:0.0:1787828974.439231:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000020:00000001:2.0:1787828974.439236:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00080000:0.0:1787828974.439237:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828974.439243:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:2.0:1787828974.439243:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787828974.439245:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000001:2.0:1787828974.439248:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00100000:0.0:1787828974.439251:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2da40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646783360:192.168.203.143@tcp:250:kworker.0 00000100:00080000:2.0:1787828974.439256:0:40701:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787828974.439261:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828974.439262:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828974.439268:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 02000000:00000001:2.0:1787828974.439268:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787828974.439271:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787828974.439272:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787828974.439292:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828974.439293:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 00000100:00000001:2.0:1787828974.439295:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 02000000:00000001:0.0:1787828974.439299:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000008:00100000:2.0:1787828974.439300:0:40701:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000020:00000001:0.0:1787828974.439303:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000008:00000001:2.0:1787828974.439305:0:40701:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787828974.439307:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 00000100:00000001:2.0:1787828974.439309:0:40701:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828974.439313:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787828974.439316:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 02000000:00000001:2.0:1787828974.439316:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:0.0:1787828974.439319:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 02000000:00000001:2.0:1787828974.439320:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828974.439323:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828974.439325:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:0.0:1787828974.439327:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:2.0:1787828974.439328:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787828974.439332:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828974.439336:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787828974.439339:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828974.439339:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828974.439344:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787828974.439347:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787828974.439348:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1787828974.439350:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00100000:2.0:1787828974.439351:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2ed00 x1874670646783744/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787828974.439352:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828974.439356:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787828974.439359:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787828974.439363:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828974.439370:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646783488:192.168.203.143@tcp:38:kworker.0 00000100:00000001:2.0:1787828974.439372:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787828974.439375:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787828974.439379:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:2.0:1787828974.439379:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828974.439384:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828974.439387:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:2.0:1787828974.439387:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828974.439390:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828974.439398:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787828974.439399:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417881728 : -126234291669888 : ffff8d30c9e2f480) 00000100:00000001:0.0:1787828974.439403:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1787828974.439421:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:0.0:1787828974.439426:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1787828974.439429:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828974.439430:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828974.439432:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787828974.439435:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828974.439441:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298447382 00000100:00000001:0.0:1787828974.439444:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298448640 00000100:00000001:0.0:1787828974.439447:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298447382 00000100:00000001:0.0:1787828974.439450:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298448640 00000100:00000001:0.0:1787828974.439453:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787828974.439455:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828974.439457:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787828974.439462:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828974/real 1787828974] req@ffff8d30c9e2f480 x1874670646783488/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828999 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787828974.439472:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787828974.439473:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787828974.439476:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787828974.439478:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787828974.439479:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828974.439481:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828974.439483:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787828974.439487:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2f480 x1874670646783488/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828999 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787828974.439495:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828974.439498:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787828974.439501:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828974.439502:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828974.439504:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828974.439505:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787828974.439507:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787828974.439509:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828974.439511:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787828974.439513:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787828974.439515:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646783488:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787828974.439518:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1787828974.439519:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787828974.439520:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828974.439522:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787828974.439523:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787828974.439524:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828974.439526:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646783616:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787828974.439530:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787828974.439532:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787828974.439534:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828974.439536:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1787828974.439539:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828974.439545:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787828974.439546:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417868288 : -126234291683328 : ffff8d30c9e2c000) 00000100:00000001:0.0:1787828974.439550:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1787828974.439563:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:0.0:1787828974.439568:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1787828974.439571:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828974.439572:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828974.439574:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787828974.439577:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828974.439581:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298447382 00000100:00000001:0.0:1787828974.439583:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298448640 00000100:00000001:0.0:1787828974.439584:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298447382 00000100:00000001:0.0:1787828974.439586:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298448640 00000100:00000001:0.0:1787828974.439587:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787828974.439589:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828974.439590:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787828974.439593:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828974/real 1787828974] req@ffff8d30c9e2c000 x1874670646783616/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828999 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787828974.439618:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787828974.439619:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787828974.439622:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787828974.439624:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787828974.439625:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828974.439627:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828974.439629:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787828974.439634:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2c000 x1874670646783616/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787828999 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787828974.439641:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828974.439643:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787828974.439646:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828974.439647:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828974.439649:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828974.439651:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787828974.439652:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787828974.439655:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828974.439659:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787828974.439661:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787828974.439665:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646783616:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787828974.439669:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828974.439671:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828974.439672:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787828974.439674:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787828974.439676:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787828974.439677:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787828974.439678:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787828974.439680:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787828974.439681:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787828974.439683:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787828974.439684:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828974.439685:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828974.439686:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787828974.439687:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787828974.439688:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787828974.439689:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787828974.439689:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787828974.439690:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787828974.439691:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787828974.439693:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787828974.439694:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828974.439695:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828974.439698:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787828974.439699:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828974.439701:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787828974.439702:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787828974.439703:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1787828974.439704:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787828974.439705:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828974.439707:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787828974.439708:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787828974.439709:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828974.439712:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646783744:192.168.203.143@tcp:8:kworker.0 00000100:00000001:0.0:1787828974.439714:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787828974.439715:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787828974.439717:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828974.439718:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1787828974.439720:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828974.439727:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787828974.439728:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417879808 : -126234291671808 : ffff8d30c9e2ed00) 00000100:00000001:0.0:1787828974.439732:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1787828974.439745:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:0.0:1787828974.439750:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1787828974.439752:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828974.439754:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828974.439756:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787828974.439759:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828974.439762:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298447396 00000100:00000001:0.0:1787828974.439765:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298448640 00000100:00000001:0.0:1787828974.439768:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298447396 00000100:00000001:0.0:1787828974.439770:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298448640 00000100:00000001:0.0:1787828974.439772:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787828974.439774:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828974.439776:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787828974.439779:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828974/real 1787828974] req@ffff8d30c9e2ed00 x1874670646783744/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828999 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787828974.439785:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787828974.439786:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787828974.439788:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787828974.439789:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787828974.439790:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828974.439792:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828974.439793:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787828974.439795:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2ed00 x1874670646783744/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787828999 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787828974.439800:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828974.439801:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787828974.439803:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828974.439804:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828974.439805:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828974.439806:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787828974.439807:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787828974.439808:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828974.439811:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787828974.439812:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787828974.439814:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646783744:192.168.203.143@tcp:8:kworker.0 00000100:00000001:0.0:1787828974.439817:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828974.439818:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828974.439819:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787828974.439820:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787828974.439821:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787828974.439822:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787828974.439823:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787828974.439824:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787828974.439825:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787828974.439828:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787828974.439829:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828974.439830:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828974.439832:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787828974.439833:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828974.439834:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787828974.439836:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828975.334675:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828975.334680:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828975.334730:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828975.334734:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828975.334738:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828976.358688:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828976.358694:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828976.358755:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828976.358759:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828976.358762:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828977.382699:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828977.382704:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828977.382784:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828977.382788:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828977.382791:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828978.406691:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828978.406697:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828978.406787:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828978.406791:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828978.406793:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828979.430670:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828979.430675:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828979.430750:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828979.430754:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828979.430756:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787828979.558704:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828979.558712:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1787828979.558714:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1787828979.558717:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787828979.558721:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1787828979.558726:0:40701:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 3780 00000100:00080000:1.0:1787828979.558734:0:40701:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1787828979.558736:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787828979.558738:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:1.0:1787828979.558741:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:1.0:1787828979.558743:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:1.0:1787828979.558745:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:1.0:1787828979.558747:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:1.0:1787828979.558748:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787828979.558749:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:1.0:1787828979.558753:0:40701:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:1.0:1787828979.558755:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828979.558756:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1787828979.558759:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787828979.558762:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828979.558763:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:1.0:1787828979.558765:0:40701:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828979.558774:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1787828979.558775:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828979.558778:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1787828979.558780:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1787828979.558784:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828979.558792:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787828979.558800:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d13ef0c0 x1874670646783872/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787828979.558807:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787828979.558810:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828979.558824:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1787828979.558826:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828979.558829:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1787828979.558830:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1787828979.558831:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787828979.558833:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1787828979.558835:0:40701:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3780 00000100:00080000:1.0:1787828979.558837:0:40701:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1787828979.558838:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787828979.558839:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:1.0:1787828979.558841:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:1.0:1787828979.558842:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:1.0:1787828979.558843:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000020:00000001:1.0:1787828979.558845:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:1.0:1787828979.558847:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787828979.558848:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:1.0:1787828979.558851:0:40701:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:1.0:1787828979.558852:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828979.558854:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1787828979.558859:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787828979.558860:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:1.0:1787828979.558861:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828979.558865:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1787828979.558866:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787828979.558870:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000008:00100000:1.0:1787828979.558870:0:40701:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00000001:0.0:1787828979.558875:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000008:00000001:1.0:1787828979.558876:0:40701:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828979.558879:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828979.558881:0:40701:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828979.558885:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787828979.558888:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787828979.558890:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787828979.558892:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828979.558897:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828979.558898:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1787828979.558901:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1787828979.558905:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828979.558907:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d13ef0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646783872:192.168.203.143@tcp:250:kworker.0 00000100:00000001:1.0:1787828979.558912:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828979.558914:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787828979.558921:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000100:00100000:1.0:1787828979.558923:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d13efc00 x1874670646784000/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:0.0:1787828979.558925:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828979.558931:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:1.0:1787828979.558936:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 02000000:00000001:0.0:1787828979.558937:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828979.558939:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828979.558944:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1787828979.558946:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787828979.558949:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:1.0:1787828979.558950:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1787828979.558951:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787828979.558953:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787828979.558954:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:0.0:1787828979.558956:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839541350592 : -126234168201024 : ffff8d30d13ef0c0) 00000100:00080000:1.0:1787828979.558958:0:40701:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3780 00000100:00000001:0.0:1787828979.558961:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00080000:1.0:1787828979.558964:0:40701:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1787828979.558967:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787828979.558968:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:1.0:1787828979.558969:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:1.0:1787828979.558971:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:1.0:1787828979.558972:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000020:00000001:1.0:1787828979.558974:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:1.0:1787828979.558976:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787828979.558978:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:1.0:1787828979.558981:0:40701:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:1.0:1787828979.558983:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828979.558986:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1787828979.558989:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787828979.558991:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828979.558994:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000800:00000100:0.0:1787828979.558997:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000008:00100000:1.0:1787828979.558998:0:40701:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1787828979.559003:0:40701:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828979.559006:0:40701:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828979.559008:0:2390:0:(events.c:34:request_out_callback()) Process entered 02000000:00000001:1.0:1787828979.559012:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:0.0:1787828979.559013:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 02000000:00000001:1.0:1787828979.559016:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828979.559017:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828979.559023:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787828979.559023:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1787828979.559028:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1787828979.559031:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828979.559035:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828979.559038:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787828979.559041:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d13ed2c0 x1874670646784128/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787828979.559043:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4298453760 00000100:00000001:1.0:1787828979.559051:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787828979.559053:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828979.559055:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 00000100:00000001:1.0:1787828979.559056:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1787828979.559061:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1787828979.559063:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4298453760 00000100:00080000:1.0:1787828979.559064:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1787828979.559067:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:0.0:1787828979.559070:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 00000100:00080000:1.0:1787828979.559070:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787828979.559076:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:0.0:1787828979.559077:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00080000:1.0:1787828979.559080:0:40701:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3780 00000100:00000001:0.0:1787828979.559082:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1787828979.559085:0:40701:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787828979.559086:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00000001:1.0:1787828979.559089:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787828979.559092:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00100000:0.0:1787828979.559093:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828979/real 1787828979] req@ffff8d30d13ef0c0 x1874670646783872/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829004 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:1.0:1787828979.559094:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:1.0:1787828979.559097:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:1.0:1787828979.559100:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000020:00000001:1.0:1787828979.559104:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:1.0:1787828979.559110:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000001:0.0:1787828979.559112:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000001:1.0:1787828979.559112:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000400:00000001:0.0:1787828979.559115:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00080000:1.0:1787828979.559118:0:40701:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787828979.559120:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787828979.559121:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828979.559123:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 02000000:00000001:1.0:1787828979.559124:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:0.0:1787828979.559128:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:1.0:1787828979.559129:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787828979.559132:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 02000000:00000001:1.0:1787828979.559133:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828979.559135:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828979.559136:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:1.0:1787828979.559139:0:40701:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:0.0:1787828979.559142:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d13ef0c0 x1874670646783872/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829004 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00000001:1.0:1787828979.559142:0:40701:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828979.559147:0:40701:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787828979.559153:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:0.0:1787828979.559154:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:1.0:1787828979.559155:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828979.559159:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:0.0:1787828979.559160:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 02000000:00000001:1.0:1787828979.559161:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1787828979.559165:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828979.559166:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828979.559169:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828979.559170:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787828979.559170:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828979.559177:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787828979.559179:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00100000:1.0:1787828979.559180:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d13ed680 x1874670646784256/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1787828979.559182:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828979.559190:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787828979.559194:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000001:1.0:1787828979.559196:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00100000:0.0:1787828979.559200:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d13ef0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646783872:192.168.203.143@tcp:250:kworker.0 00000100:00000001:1.0:1787828979.559201:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787828979.559205:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828979.559208:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828979.559211:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828979.559213:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787828979.559214:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787828979.559216:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787828979.559218:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787828979.559219:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787828979.559223:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787828979.559225:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787828979.559227:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787828979.559228:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828979.559229:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828979.559235:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787828979.559236:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828979.559238:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787828979.559239:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787828979.559253:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1787828979.559255:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787828979.559256:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828979.559258:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787828979.559260:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787828979.559261:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828979.559264:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d13efc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646784000:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787828979.559267:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787828979.559268:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787828979.559269:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828979.559271:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1787828979.559273:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828979.559277:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787828979.559278:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839541353472 : -126234168198144 : ffff8d30d13efc00) 00000100:00000001:0.0:1787828979.559281:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1787828979.559291:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:0.0:1787828979.559294:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1787828979.559296:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828979.559297:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828979.559298:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787828979.559300:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828979.559303:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298452502 00000100:00000001:0.0:1787828979.559306:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298453760 00000100:00000001:0.0:1787828979.559307:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298452502 00000100:00000001:0.0:1787828979.559309:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298453760 00000100:00000001:0.0:1787828979.559310:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787828979.559311:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828979.559312:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787828979.559315:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828979/real 1787828979] req@ffff8d30d13efc00 x1874670646784000/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829004 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787828979.559321:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787828979.559321:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787828979.559323:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787828979.559324:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787828979.559325:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828979.559326:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828979.559327:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787828979.559330:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d13efc00 x1874670646784000/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829004 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787828979.559334:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828979.559336:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787828979.559337:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828979.559338:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828979.559339:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828979.559340:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787828979.559341:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787828979.559343:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828979.559345:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787828979.559346:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787828979.559348:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d13efc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646784000:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787828979.559350:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1787828979.559351:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787828979.559352:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828979.559353:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787828979.559355:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787828979.559356:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828979.559358:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d13ed2c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646784128:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787828979.559360:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787828979.559361:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787828979.559362:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828979.559363:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1787828979.559365:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828979.559367:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787828979.559368:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839541342912 : -126234168208704 : ffff8d30d13ed2c0) 00000100:00000001:0.0:1787828979.559371:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1787828979.559377:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:0.0:1787828979.559379:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1787828979.559381:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828979.559382:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828979.559383:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787828979.559385:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828979.559387:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298452502 00000100:00000001:0.0:1787828979.559388:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298453760 00000100:00000001:0.0:1787828979.559390:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298452502 00000100:00000001:0.0:1787828979.559391:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298453760 00000100:00000001:0.0:1787828979.559393:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787828979.559394:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828979.559395:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787828979.559397:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828979/real 1787828979] req@ffff8d30d13ed2c0 x1874670646784128/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829004 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787828979.559402:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787828979.559402:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787828979.559404:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787828979.559404:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787828979.559405:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828979.559406:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828979.559407:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787828979.559409:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d13ed2c0 x1874670646784128/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829004 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787828979.559414:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828979.559415:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787828979.559417:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828979.559418:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828979.559419:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828979.559420:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787828979.559421:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787828979.559422:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828979.559424:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787828979.559425:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787828979.559427:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d13ed2c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646784128:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787828979.559430:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1787828979.559431:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787828979.559432:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828979.559433:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787828979.559434:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787828979.559435:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828979.559437:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d13ed680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646784256:192.168.203.143@tcp:8:kworker.0 00000100:00000001:0.0:1787828979.559439:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787828979.559440:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787828979.559441:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828979.559442:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1787828979.559443:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828979.559446:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787828979.559447:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839541343872 : -126234168207744 : ffff8d30d13ed680) 00000100:00000001:0.0:1787828979.559448:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1787828979.559454:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:0.0:1787828979.559456:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1787828979.559457:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828979.559458:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828979.559459:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787828979.559461:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828979.559470:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298452515 00000100:00000001:0.0:1787828979.559473:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298453760 00000100:00000001:0.0:1787828979.559474:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298452515 00000100:00000001:0.0:1787828979.559476:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298453760 00000100:00000001:0.0:1787828979.559477:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787828979.559481:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828979.559482:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787828979.559484:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828979/real 1787828979] req@ffff8d30d13ed680 x1874670646784256/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829004 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787828979.559495:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787828979.559496:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787828979.559497:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787828979.559498:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787828979.559499:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828979.559500:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828979.559501:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787828979.559508:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d13ed680 x1874670646784256/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829004 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787828979.559514:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828979.559515:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787828979.559517:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828979.559518:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828979.559519:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828979.559520:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787828979.559521:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787828979.559522:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828979.559524:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787828979.559525:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787828979.559527:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d13ed680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646784256:192.168.203.143@tcp:8:kworker.0 00000100:00000001:0.0:1787828979.559530:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828979.559531:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828979.559533:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787828979.559534:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787828979.559535:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787828979.559537:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787828979.559537:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787828979.559539:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787828979.559540:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787828979.559541:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787828979.559542:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828979.559544:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828979.559545:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787828979.559546:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787828979.559547:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787828979.559548:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787828979.559549:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787828979.559551:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787828979.559551:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787828979.559552:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787828979.559553:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828979.559554:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828979.559555:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787828979.559555:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787828979.559556:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787828979.559557:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787828979.559558:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787828979.559559:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787828979.559559:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787828979.559560:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787828979.559561:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828979.559562:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828979.559564:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787828979.559565:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828979.559566:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787828979.559567:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828980.454692:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828980.454697:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828980.454793:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828980.454798:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828980.454802:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828981.478706:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828981.478713:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828981.478777:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828981.478783:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828981.478786:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828982.502672:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828982.502686:0:815:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1787828982.502688:0:815:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1787828982.502691:0:815:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828982.502694:0:815:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1787828982.502696:0:815:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828982.502698:0:815:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828982.502699:0:815:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828982.502751:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828982.502755:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828982.502758:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828983.526713:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828983.526720:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828983.526762:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828983.526765:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828983.526768:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828984.550689:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828984.550695:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828984.550755:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828984.550758:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828984.550761:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787828984.678690:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828984.678701:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1787828984.678703:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787828984.678706:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787828984.678710:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787828984.678715:0:468:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 3785 00000100:00080000:3.0:1787828984.678724:0:468:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787828984.678727:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787828984.678729:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:3.0:1787828984.678732:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1787828984.678734:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1787828984.678736:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:3.0:1787828984.678738:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:3.0:1787828984.678740:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787828984.678741:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:3.0:1787828984.678744:0:468:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:3.0:1787828984.678746:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787828984.678748:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787828984.678751:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787828984.678754:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828984.678755:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:3.0:1787828984.678757:0:468:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787828984.678767:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787828984.678768:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828984.678770:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787828984.678773:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787828984.678776:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828984.678786:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787828984.678796:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c70170c0 x1874670646784384/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787828984.678807:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787828984.678811:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828984.678828:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787828984.678831:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828984.678835:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1787828984.678837:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787828984.678838:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787828984.678841:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787828984.678843:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3785 00000100:00080000:3.0:1787828984.678846:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787828984.678848:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787828984.678850:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:3.0:1787828984.678852:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1787828984.678853:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1787828984.678855:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000020:00000001:3.0:1787828984.678857:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:3.0:1787828984.678858:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787828984.678860:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:3.0:1787828984.678863:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787828984.678864:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1787828984.678865:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787828984.678869:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:0.0:1787828984.678871:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787828984.678874:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 02000000:00000001:3.0:1787828984.678875:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787828984.678876:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787828984.678879:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787828984.678880:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828984.678884:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 02000000:00000001:0.0:1787828984.678886:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000008:00100000:3.0:1787828984.678888:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 02000000:00000001:0.0:1787828984.678890:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000008:00000001:3.0:1787828984.678893:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828984.678894:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828984.678896:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828984.678901:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c70170c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646784384:192.168.203.143@tcp:250:kworker.0 02000000:00000001:3.0:1787828984.678901:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787828984.678903:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828984.678908:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:0.0:1787828984.678909:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1787828984.678910:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787828984.678914:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1787828984.678914:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828984.678917:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828984.678919:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828984.678920:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1787828984.678925:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787828984.678929:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c7016d00 x1874670646784512/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787828984.678940:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787828984.678940:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787828984.678944:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828984.678945:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369547968 : -126234340003648 : ffff8d30c70170c0) 00000100:00000001:0.0:1787828984.678952:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:3.0:1787828984.678953:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787828984.678958:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828984.678963:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1787828984.678965:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787828984.678967:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787828984.678970:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787828984.678973:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3785 00000100:00080000:3.0:1787828984.678978:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787828984.678979:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787828984.678981:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000800:00000100:0.0:1787828984.678985:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000020:00000001:3.0:1787828984.678986:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1787828984.678989:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1787828984.678990:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000020:00000001:3.0:1787828984.678992:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:3.0:1787828984.678994:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787828984.679005:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1787828984.679009:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787828984.679009:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:0.0:1787828984.679013:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828984.679015:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00080000:3.0:1787828984.679019:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787828984.679021:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828984.679023:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828984.679026:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4298458880 02000000:00000001:3.0:1787828984.679028:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787828984.679034:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787828984.679036:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 02000000:00000001:3.0:1787828984.679037:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828984.679042:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1787828984.679043:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4298458880 00000008:00100000:3.0:1787828984.679044:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00000001:0.0:1787828984.679048:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 00000008:00000001:3.0:1787828984.679049:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828984.679054:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828984.679055:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787828984.679061:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828984.679063:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 02000000:00000001:3.0:1787828984.679065:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00100000:0.0:1787828984.679068:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828984/real 1787828984] req@ffff8d30c70170c0 x1874670646784384/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829009 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1787828984.679069:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828984.679075:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787828984.679079:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787828984.679083:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:0.0:1787828984.679084:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787828984.679087:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787828984.679091:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787828984.679093:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787828984.679093:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828984.679096:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828984.679099:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828984.679100:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787828984.679104:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c7014b40 x1874670646784640/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787828984.679107:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c70170c0 x1874670646784384/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829009 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787828984.679124:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787828984.679126:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787828984.679128:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828984.679131:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787828984.679133:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828984.679137:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:3.0:1787828984.679138:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1787828984.679141:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00080000:3.0:1787828984.679143:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1787828984.679144:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787828984.679147:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:0.0:1787828984.679148:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00080000:3.0:1787828984.679150:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787828984.679151:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787828984.679154:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787828984.679156:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787828984.679159:0:468:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3785 00000100:00080000:0.0:1787828984.679163:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00080000:3.0:1787828984.679165:0:468:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787828984.679166:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000001:3.0:1787828984.679172:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00100000:0.0:1787828984.679173:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c70170c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646784384:192.168.203.143@tcp:250:kworker.0 00000100:00000001:3.0:1787828984.679176:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:0.0:1787828984.679178:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828984.679182:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000020:00000001:3.0:1787828984.679183:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:0.0:1787828984.679184:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787828984.679186:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 00000020:00000001:3.0:1787828984.679188:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 02000000:00000001:0.0:1787828984.679189:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787828984.679193:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787828984.679194:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000020:00000001:0.0:1787828984.679195:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787828984.679199:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 00000020:00000001:3.0:1787828984.679200:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 02000000:00000001:0.0:1787828984.679202:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787828984.679220:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787828984.679222:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787828984.679223:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787828984.679226:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:0.0:1787828984.679227:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828984.679235:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00080000:3.0:1787828984.679235:0:468:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787828984.679238:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828984.679242:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828984.679243:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787828984.679245:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 02000000:00000001:3.0:1787828984.679248:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:0.0:1787828984.679249:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1787828984.679252:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787828984.679254:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787828984.679255:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787828984.679258:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828984.679261:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:3.0:1787828984.679262:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 02000000:00000001:0.0:1787828984.679264:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000008:00100000:3.0:1787828984.679267:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 02000000:00000001:0.0:1787828984.679268:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828984.679274:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c7016d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646784512:192.168.203.143@tcp:38:kworker.0 00000008:00000001:3.0:1787828984.679275:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828984.679279:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:3.0:1787828984.679281:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828984.679283:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787828984.679286:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828984.679287:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1787828984.679290:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787828984.679290:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787828984.679293:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828984.679297:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787828984.679298:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:0.0:1787828984.679300:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369547008 : -126234340004608 : ffff8d30c7016d00) 02000000:00000001:3.0:1787828984.679302:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:0.0:1787828984.679305:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 02000000:00000001:3.0:1787828984.679308:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828984.679315:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:0.0:1787828984.679320:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00100000:3.0:1787828984.679322:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c70152c0 x1874670646784768/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787828984.679325:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1787828984.679328:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828984.679330:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828984.679332:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787828984.679334:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828984.679337:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298457622 00000100:00000001:3.0:1787828984.679340:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787828984.679341:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298458880 00000100:00000001:3.0:1787828984.679346:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828984.679347:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298457622 00000100:00000001:0.0:1787828984.679350:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298458880 00000100:00000001:0.0:1787828984.679353:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787828984.679355:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828984.679355:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828984.679362:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787828984.679369:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828984/real 1787828984] req@ffff8d30c7016d00 x1874670646784512/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829009 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787828984.679379:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787828984.679380:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787828984.679383:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787828984.679386:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787828984.679387:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828984.679389:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828984.679391:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787828984.679395:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c7016d00 x1874670646784512/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829009 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787828984.679403:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828984.679406:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787828984.679408:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828984.679410:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828984.679411:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828984.679413:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787828984.679415:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787828984.679418:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828984.679422:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787828984.679425:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787828984.679428:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c7016d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646784512:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787828984.679434:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1787828984.679435:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787828984.679436:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828984.679439:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787828984.679440:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787828984.679442:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828984.679445:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c7014b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646784640:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787828984.679448:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787828984.679450:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787828984.679451:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828984.679453:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1787828984.679456:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828984.679461:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787828984.679463:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369538368 : -126234340013248 : ffff8d30c7014b40) 00000100:00000001:0.0:1787828984.679466:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1787828984.679478:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:0.0:1787828984.679483:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1787828984.679485:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828984.679487:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828984.679489:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787828984.679491:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828984.679495:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298457622 00000100:00000001:0.0:1787828984.679498:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298458880 00000100:00000001:0.0:1787828984.679499:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298457622 00000100:00000001:0.0:1787828984.679502:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298458880 00000100:00000001:0.0:1787828984.679504:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787828984.679506:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828984.679508:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787828984.679512:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828984/real 1787828984] req@ffff8d30c7014b40 x1874670646784640/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829009 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787828984.679521:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787828984.679522:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787828984.679524:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787828984.679525:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787828984.679526:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828984.679527:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828984.679528:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787828984.679532:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c7014b40 x1874670646784640/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829009 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787828984.679536:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828984.679538:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787828984.679539:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828984.679540:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828984.679541:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828984.679542:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787828984.679543:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787828984.679545:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828984.679547:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787828984.679548:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787828984.679550:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c7014b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646784640:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787828984.679552:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828984.679553:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828984.679554:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787828984.679555:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787828984.679557:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787828984.679558:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787828984.679559:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787828984.679561:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787828984.679562:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787828984.679563:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787828984.679564:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828984.679566:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828984.679566:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787828984.679567:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787828984.679568:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787828984.679569:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787828984.679570:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787828984.679571:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787828984.679572:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787828984.679572:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787828984.679573:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828984.679574:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828984.679577:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787828984.679578:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828984.679579:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787828984.679580:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787828984.679581:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1787828984.679582:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787828984.679582:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828984.679584:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787828984.679585:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787828984.679586:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828984.679588:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c70152c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646784768:192.168.203.143@tcp:8:kworker.0 00000100:00000001:0.0:1787828984.679590:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787828984.679592:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787828984.679593:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828984.679594:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1787828984.679596:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828984.679622:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787828984.679623:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369540288 : -126234340011328 : ffff8d30c70152c0) 00000100:00000001:0.0:1787828984.679626:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1787828984.679636:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:0.0:1787828984.679639:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1787828984.679640:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828984.679641:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828984.679642:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787828984.679644:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828984.679647:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298457636 00000100:00000001:0.0:1787828984.679649:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298458880 00000100:00000001:0.0:1787828984.679650:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298457636 00000100:00000001:0.0:1787828984.679652:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298458880 00000100:00000001:0.0:1787828984.679654:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787828984.679655:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828984.679656:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787828984.679659:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828984/real 1787828984] req@ffff8d30c70152c0 x1874670646784768/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829009 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787828984.679663:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787828984.679664:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787828984.679666:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787828984.679667:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787828984.679668:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828984.679669:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828984.679670:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787828984.679672:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c70152c0 x1874670646784768/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829009 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787828984.679677:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828984.679678:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787828984.679680:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828984.679680:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828984.679681:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828984.679682:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787828984.679683:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787828984.679685:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828984.679687:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787828984.679688:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787828984.679690:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c70152c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646784768:192.168.203.143@tcp:8:kworker.0 00000100:00000001:0.0:1787828984.679692:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828984.679693:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828984.679694:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787828984.679695:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787828984.679697:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787828984.679698:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787828984.679698:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787828984.679700:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787828984.679701:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787828984.679702:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787828984.679703:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828984.679704:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828984.679706:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787828984.679707:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828984.679708:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787828984.679709:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828985.574658:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828985.574664:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828985.574706:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828985.574709:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828985.574712:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828986.598672:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828986.598678:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828986.598724:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828986.598727:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828986.598729:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828987.622710:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828987.622715:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828987.622774:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828987.622778:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828987.622782:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828988.646717:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828988.646722:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828988.646807:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828988.646815:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828988.646820:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828989.670738:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828989.670744:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828989.670832:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828989.670835:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828989.670838:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787828989.798715:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828989.798724:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1787828989.798725:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787828989.798728:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787828989.798732:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787828989.798738:0:468:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 3790 00000100:00080000:3.0:1787828989.798746:0:468:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787828989.798748:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787828989.798750:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:3.0:1787828989.798753:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1787828989.798755:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1787828989.798758:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:3.0:1787828989.798759:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:3.0:1787828989.798761:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787828989.798762:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:3.0:1787828989.798765:0:468:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:3.0:1787828989.798766:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787828989.798768:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787828989.798771:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787828989.798773:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828989.798774:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:3.0:1787828989.798776:0:468:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787828989.798785:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787828989.798786:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828989.798789:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787828989.798791:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787828989.798794:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828989.798805:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787828989.798812:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c7015680 x1874670646784896/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787828989.798820:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787828989.798823:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828989.798836:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787828989.798838:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828989.798841:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1787828989.798842:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787828989.798843:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787828989.798845:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787828989.798847:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3790 00000100:00080000:3.0:1787828989.798849:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787828989.798850:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787828989.798851:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:3.0:1787828989.798853:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1787828989.798854:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1787828989.798856:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000020:00000001:3.0:1787828989.798857:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:3.0:1787828989.798859:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787828989.798860:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:3.0:1787828989.798862:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:3.0:1787828989.798864:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787828989.798865:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:0.0:1787828989.798868:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:3.0:1787828989.798869:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787828989.798873:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828989.798875:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787828989.798875:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1787828989.798878:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000008:00100000:3.0:1787828989.798879:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00000001:0.0:1787828989.798880:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787828989.798885:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:3.0:1787828989.798885:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828989.798890:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828989.798892:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787828989.798896:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787828989.798896:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787828989.798899:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828989.798901:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828989.798902:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787828989.798905:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787828989.798907:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828989.798908:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c7015680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646784896:192.168.203.143@tcp:250:kworker.0 00000100:00000001:0.0:1787828989.798914:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:3.0:1787828989.798915:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828989.798919:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787828989.798924:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787828989.798925:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c70143c0 x1874670646785024/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:0.0:1787828989.798929:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1787828989.798936:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828989.798938:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787828989.798942:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828989.798947:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787828989.798950:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1787828989.798954:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00080000:3.0:1787828989.798955:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1787828989.798957:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787828989.798959:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787828989.798962:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787828989.798965:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3790 00000100:00000001:0.0:1787828989.798968:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369541248 : -126234340010368 : ffff8d30c7015680) 00000100:00080000:3.0:1787828989.798972:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787828989.798974:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:3.0:1787828989.798976:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787828989.798980:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:3.0:1787828989.798983:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1787828989.798985:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1787828989.798987:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000020:00000001:3.0:1787828989.798990:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:3.0:1787828989.798993:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787828989.798994:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:3.0:1787828989.798998:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:3.0:1787828989.799001:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787828989.799003:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787828989.799007:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787828989.799012:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828989.799015:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000800:00000100:0.0:1787828989.799018:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000008:00100000:3.0:1787828989.799019:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1787828989.799024:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828989.799026:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787828989.799032:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:0.0:1787828989.799033:0:2390:0:(events.c:34:request_out_callback()) Process entered 02000000:00000001:3.0:1787828989.799035:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828989.799039:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787828989.799039:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787828989.799042:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:0.0:1787828989.799043:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787828989.799046:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828989.799047:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787828989.799053:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828989.799056:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828989.799058:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4298464000 00000100:00100000:3.0:1787828989.799067:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c7014f00 x1874670646785152/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787828989.799068:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1787828989.799075:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4298464000 00000100:00000001:0.0:1787828989.799081:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 00000100:00000001:3.0:1787828989.799082:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787828989.799084:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787828989.799088:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828989.799089:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828989.799095:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00000001:3.0:1787828989.799096:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828989.799102:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828989/real 1787828989] req@ffff8d30c7015680 x1874670646784896/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829014 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828989.799102:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787828989.799113:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1787828989.799117:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000400:00000001:0.0:1787828989.799119:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787828989.799122:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00080000:3.0:1787828989.799122:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787828989.799127:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787828989.799128:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:0.0:1787828989.799130:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787828989.799132:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828989.799135:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00080000:3.0:1787828989.799135:0:468:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3790 00000100:00000001:0.0:1787828989.799138:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787828989.799145:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c7015680 x1874670646784896/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829014 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787828989.799145:0:468:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787828989.799152:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787828989.799155:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:0.0:1787828989.799156:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828989.799163:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000020:00000001:3.0:1787828989.799163:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1787828989.799167:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:0.0:1787828989.799168:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828989.799184:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828989.799186:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:3.0:1787828989.799188:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:0.0:1787828989.799189:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787828989.799191:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000020:00000001:3.0:1787828989.799195:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00080000:0.0:1787828989.799196:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828989.799201:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787828989.799202:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787828989.799204:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000001:3.0:1787828989.799207:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00100000:0.0:1787828989.799209:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c7015680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646784896:192.168.203.143@tcp:250:kworker.0 00000100:00080000:3.0:1787828989.799216:0:468:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787828989.799217:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828989.799221:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787828989.799223:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828989.799224:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787828989.799227:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787828989.799228:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787828989.799231:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787828989.799234:0:2390:0:(genops.c:1217:class_import_put()) Process entered 02000000:00000001:3.0:1787828989.799234:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000020:00000001:0.0:1787828989.799237:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787828989.799241:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828989.799242:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787828989.799246:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787828989.799246:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:3.0:1787828989.799251:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00000001:0.0:1787828989.799254:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000008:00000001:3.0:1787828989.799256:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828989.799257:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828989.799263:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787828989.799263:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787828989.799273:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:0.0:1787828989.799274:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787828989.799278:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787828989.799278:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828989.799286:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1787828989.799286:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787828989.799290:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:0.0:1787828989.799291:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787828989.799295:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 02000000:00000001:3.0:1787828989.799296:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828989.799297:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787828989.799299:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828989.799303:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787828989.799305:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787828989.799307:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828989.799307:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828989.799315:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c70143c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646785024:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787828989.799320:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00100000:3.0:1787828989.799321:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c7015a40 x1874670646785280/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:0.0:1787828989.799322:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787828989.799325:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828989.799328:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1787828989.799331:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787828989.799338:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787828989.799339:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787828989.799342:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828989.799343:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369536448 : -126234340015168 : ffff8d30c70143c0) 00000100:00000001:0.0:1787828989.799350:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:3.0:1787828989.799352:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:0.0:1787828989.799364:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:0.0:1787828989.799369:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1787828989.799373:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828989.799374:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828989.799375:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787828989.799377:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828989.799381:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298462742 00000100:00000001:0.0:1787828989.799383:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298464000 00000100:00000001:0.0:1787828989.799385:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298462742 00000100:00000001:0.0:1787828989.799387:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298464000 00000100:00000001:0.0:1787828989.799389:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787828989.799390:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828989.799391:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787828989.799394:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828989/real 1787828989] req@ffff8d30c70143c0 x1874670646785024/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829014 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787828989.799400:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787828989.799401:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787828989.799402:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787828989.799403:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787828989.799404:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828989.799405:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828989.799406:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787828989.799409:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c70143c0 x1874670646785024/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829014 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787828989.799414:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828989.799415:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787828989.799416:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828989.799417:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828989.799418:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828989.799419:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787828989.799420:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787828989.799422:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828989.799424:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787828989.799425:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787828989.799427:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c70143c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646785024:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787828989.799429:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1787828989.799430:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787828989.799431:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828989.799433:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787828989.799434:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787828989.799436:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828989.799438:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c7014f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646785152:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787828989.799440:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787828989.799441:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787828989.799442:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828989.799443:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1787828989.799445:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828989.799448:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787828989.799449:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369539328 : -126234340012288 : ffff8d30c7014f00) 00000100:00000001:0.0:1787828989.799451:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1787828989.799458:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:0.0:1787828989.799460:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1787828989.799461:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828989.799462:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828989.799463:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787828989.799465:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828989.799467:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298462742 00000100:00000001:0.0:1787828989.799469:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298464000 00000100:00000001:0.0:1787828989.799470:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298462742 00000100:00000001:0.0:1787828989.799471:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298464000 00000100:00000001:0.0:1787828989.799473:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787828989.799474:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828989.799475:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787828989.799477:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828989/real 1787828989] req@ffff8d30c7014f00 x1874670646785152/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829014 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787828989.799482:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787828989.799482:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787828989.799484:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787828989.799485:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787828989.799485:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828989.799486:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828989.799487:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787828989.799490:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c7014f00 x1874670646785152/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829014 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787828989.799495:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828989.799496:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787828989.799497:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828989.799498:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828989.799499:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828989.799500:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787828989.799501:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787828989.799502:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828989.799504:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787828989.799505:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787828989.799507:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c7014f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646785152:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787828989.799509:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828989.799510:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828989.799511:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787828989.799512:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787828989.799513:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787828989.799514:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787828989.799515:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787828989.799516:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787828989.799523:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787828989.799525:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787828989.799526:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828989.799527:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828989.799527:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787828989.799528:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787828989.799529:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787828989.799530:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787828989.799531:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787828989.799532:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787828989.799533:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787828989.799534:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787828989.799535:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828989.799536:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828989.799538:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787828989.799539:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828989.799541:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787828989.799542:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787828989.799543:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1787828989.799544:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787828989.799545:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828989.799546:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787828989.799547:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787828989.799548:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828989.799550:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c7015a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646785280:192.168.203.143@tcp:8:kworker.0 00000100:00000001:0.0:1787828989.799553:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787828989.799554:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787828989.799556:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828989.799557:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1787828989.799559:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828989.799562:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787828989.799563:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369542208 : -126234340009408 : ffff8d30c7015a40) 00000100:00000001:0.0:1787828989.799566:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1787828989.799572:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:0.0:1787828989.799575:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1787828989.799576:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828989.799577:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828989.799578:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787828989.799580:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828989.799582:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298462755 00000100:00000001:0.0:1787828989.799584:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298464000 00000100:00000001:0.0:1787828989.799586:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298462755 00000100:00000001:0.0:1787828989.799588:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298464000 00000100:00000001:0.0:1787828989.799589:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787828989.799590:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828989.799591:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787828989.799594:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828989/real 1787828989] req@ffff8d30c7015a40 x1874670646785280/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829014 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787828989.799623:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787828989.799624:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787828989.799626:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787828989.799627:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787828989.799628:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828989.799629:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828989.799630:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787828989.799633:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c7015a40 x1874670646785280/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829014 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787828989.799637:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828989.799639:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787828989.799641:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828989.799642:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828989.799643:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828989.799644:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787828989.799645:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787828989.799646:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828989.799648:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787828989.799649:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787828989.799651:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c7015a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646785280:192.168.203.143@tcp:8:kworker.0 00000100:00000001:0.0:1787828989.799654:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828989.799655:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828989.799656:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787828989.799657:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787828989.799658:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787828989.799659:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787828989.799660:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787828989.799661:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787828989.799662:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787828989.799663:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787828989.799664:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828989.799666:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828989.799667:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787828989.799668:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828989.799670:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787828989.799671:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828990.694669:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828990.694674:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828990.694750:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828990.694753:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828990.694756:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828991.718748:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828991.718754:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828991.718813:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828991.718818:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828991.718823:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828992.742694:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828992.742701:0:815:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1787828992.742703:0:815:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1787828992.742706:0:815:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828992.742710:0:815:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1787828992.742712:0:815:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828992.742713:0:815:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828992.742715:0:815:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828992.742769:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828992.742773:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828992.742777:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828993.766694:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828993.766700:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828993.766766:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828993.766771:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828993.766775:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828994.790725:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828994.790732:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828994.790781:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828994.790784:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828994.790787:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787828994.918714:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828994.918725:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1787828994.918728:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787828994.918732:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787828994.918736:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787828994.918744:0:40701:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 3795 00000100:00080000:2.0:1787828994.918754:0:40701:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787828994.918759:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828994.918761:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787828994.918765:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787828994.918767:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787828994.918771:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:2.0:1787828994.918772:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:2.0:1787828994.918774:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828994.918775:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:2.0:1787828994.918778:0:40701:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787828994.918780:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828994.918782:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787828994.918785:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828994.918787:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828994.918789:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:2.0:1787828994.918791:0:40701:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828994.918800:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787828994.918802:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828994.918805:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787828994.918807:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787828994.918810:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828994.918818:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787828994.918825:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2f0c0 x1874670646785408/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787828994.918833:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787828994.918837:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828994.918849:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787828994.918851:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828994.918853:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1787828994.918854:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787828994.918855:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787828994.918857:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787828994.918859:0:40701:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3795 00000100:00080000:2.0:1787828994.918862:0:40701:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787828994.918864:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828994.918865:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787828994.918867:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787828994.918868:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787828994.918869:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000020:00000001:2.0:1787828994.918871:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:2.0:1787828994.918873:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828994.918874:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:2.0:1787828994.918876:0:40701:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787828994.918878:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828994.918879:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787828994.918884:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787828994.918885:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:2.0:1787828994.918886:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828994.918889:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1787828994.918891:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000008:00100000:2.0:1787828994.918893:0:40701:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00000001:0.0:1787828994.918894:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000008:00000001:2.0:1787828994.918898:0:40701:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828994.918899:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787828994.918903:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828994.918903:0:40701:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828994.918907:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787828994.918909:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787828994.918910:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828994.918914:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787828994.918915:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787828994.918917:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787828994.918919:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828994.918921:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828994.918926:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646785408:192.168.203.143@tcp:250:kworker.0 00000100:00000001:2.0:1787828994.918926:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828994.918933:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00100000:2.0:1787828994.918937:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2e940 x1874670646785536/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:0.0:1787828994.918938:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787828994.918941:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828994.918945:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1787828994.918950:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828994.918952:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787828994.918956:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828994.918961:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787828994.918964:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1787828994.918965:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787828994.918968:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417880768 : -126234291670848 : ffff8d30c9e2f0c0) 00000100:00080000:2.0:1787828994.918970:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1787828994.918972:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:2.0:1787828994.918976:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787828994.918980:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787828994.918983:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787828994.918986:0:40701:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3795 00000100:00080000:2.0:1787828994.918990:0:40701:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787828994.918992:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828994.918993:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787828994.918995:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787828994.918996:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787828994.918998:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000020:00000001:2.0:1787828994.919000:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:2.0:1787828994.919001:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828994.919003:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000800:00000100:0.0:1787828994.919006:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00080000:2.0:1787828994.919010:0:40701:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787828994.919018:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1787828994.919019:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828994.919023:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 02000000:00000001:2.0:1787828994.919025:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:0.0:1787828994.919026:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828994.919030:0:2390:0:(events.c:68:request_out_callback()) Process leaving 02000000:00000001:2.0:1787828994.919030:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787828994.919034:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828994.919036:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1787828994.919037:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000008:00100000:2.0:1787828994.919038:0:40701:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00000001:0.0:1787828994.919041:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4298469120 00000008:00000001:2.0:1787828994.919042:0:40701:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828994.919047:0:40701:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828994.919051:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:0.0:1787828994.919052:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 02000000:00000001:2.0:1787828994.919053:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828994.919057:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787828994.919070:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:0.0:1787828994.919072:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4298469120 02000000:00000001:2.0:1787828994.919073:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828994.919081:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 00000100:00000001:2.0:1787828994.919082:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828994.919085:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00100000:2.0:1787828994.919089:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2c780 x1874670646785664/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787828994.919090:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828994.919094:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00000001:2.0:1787828994.919100:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00100000:0.0:1787828994.919101:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828994/real 1787828994] req@ffff8d30c9e2f0c0 x1874670646785408/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829019 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787828994.919103:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828994.919108:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787828994.919112:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787828994.919120:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000400:00000001:0.0:1787828994.919123:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000001:2.0:1787828994.919123:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787828994.919126:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000400:00000001:0.0:1787828994.919127:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787828994.919131:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1787828994.919131:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787828994.919147:0:40701:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3795 00000100:00000001:0.0:1787828994.919150:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00080000:2.0:1787828994.919152:0:40701:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787828994.919154:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787828994.919155:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787828994.919168:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787828994.919169:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:0.0:1787828994.919171:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:2.0:1787828994.919173:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787828994.919175:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787828994.919178:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00100000:0.0:1787828994.919179:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2f0c0 x1874670646785408/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829019 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:2.0:1787828994.919181:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:2.0:1787828994.919186:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787828994.919188:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:0.0:1787828994.919194:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:2.0:1787828994.919195:0:40701:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787828994.919198:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787828994.919199:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787828994.919204:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:0.0:1787828994.919205:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:2.0:1787828994.919207:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787828994.919208:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828994.919210:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:2.0:1787828994.919212:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828994.919213:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787828994.919215:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1787828994.919216:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000008:00100000:2.0:1787828994.919218:0:40701:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00080000:0.0:1787828994.919220:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000008:00000001:2.0:1787828994.919222:0:40701:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787828994.919226:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:2.0:1787828994.919226:0:40701:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828994.919231:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 02000000:00000001:2.0:1787828994.919233:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00100000:0.0:1787828994.919237:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646785408:192.168.203.143@tcp:250:kworker.0 02000000:00000001:2.0:1787828994.919237:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787828994.919241:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:0.0:1787828994.919244:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:2.0:1787828994.919244:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787828994.919246:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828994.919247:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828994.919250:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787828994.919252:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 00000100:00000001:2.0:1787828994.919252:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828994.919257:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787828994.919261:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000100:00100000:2.0:1787828994.919261:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2de00 x1874670646785792/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000020:00000001:0.0:1787828994.919264:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787828994.919269:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 00000100:00000001:2.0:1787828994.919270:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 02000000:00000001:0.0:1787828994.919271:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1787828994.919274:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828994.919276:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787828994.919277:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828994.919279:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828994.919282:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828994.919289:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787828994.919290:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828994.919292:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787828994.919293:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787828994.919294:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1787828994.919295:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787828994.919297:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828994.919299:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787828994.919300:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787828994.919302:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828994.919306:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646785536:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787828994.919308:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787828994.919310:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787828994.919311:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828994.919312:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1787828994.919314:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828994.919319:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787828994.919320:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417878848 : -126234291672768 : ffff8d30c9e2e940) 00000100:00000001:0.0:1787828994.919323:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1787828994.919332:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:0.0:1787828994.919335:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1787828994.919337:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828994.919338:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828994.919339:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787828994.919341:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828994.919344:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298467862 00000100:00000001:0.0:1787828994.919346:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298469120 00000100:00000001:0.0:1787828994.919348:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298467862 00000100:00000001:0.0:1787828994.919349:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298469120 00000100:00000001:0.0:1787828994.919351:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787828994.919352:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828994.919353:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787828994.919356:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828994/real 1787828994] req@ffff8d30c9e2e940 x1874670646785536/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829019 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787828994.919367:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787828994.919368:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787828994.919369:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787828994.919370:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787828994.919379:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828994.919380:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828994.919381:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787828994.919384:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2e940 x1874670646785536/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829019 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787828994.919389:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828994.919391:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787828994.919392:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828994.919393:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828994.919394:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828994.919397:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787828994.919398:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787828994.919400:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828994.919402:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787828994.919403:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787828994.919405:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646785536:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787828994.919408:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1787828994.919409:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787828994.919410:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828994.919411:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787828994.919413:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787828994.919414:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828994.919416:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646785664:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787828994.919418:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787828994.919419:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787828994.919421:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828994.919422:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1787828994.919424:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828994.919426:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787828994.919427:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417870208 : -126234291681408 : ffff8d30c9e2c780) 00000100:00000001:0.0:1787828994.919430:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1787828994.919436:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:0.0:1787828994.919439:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1787828994.919441:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828994.919442:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828994.919443:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787828994.919444:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828994.919446:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298467862 00000100:00000001:0.0:1787828994.919448:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298469120 00000100:00000001:0.0:1787828994.919449:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298467862 00000100:00000001:0.0:1787828994.919451:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298469120 00000100:00000001:0.0:1787828994.919452:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787828994.919454:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828994.919455:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787828994.919457:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828994/real 1787828994] req@ffff8d30c9e2c780 x1874670646785664/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829019 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787828994.919462:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787828994.919463:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787828994.919471:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787828994.919472:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787828994.919473:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828994.919474:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828994.919477:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787828994.919480:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2c780 x1874670646785664/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829019 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787828994.919485:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828994.919486:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787828994.919488:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828994.919489:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828994.919490:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828994.919492:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787828994.919493:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787828994.919495:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828994.919496:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787828994.919498:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787828994.919499:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646785664:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787828994.919502:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1787828994.919502:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787828994.919503:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828994.919505:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787828994.919506:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787828994.919507:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787828994.919509:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646785792:192.168.203.143@tcp:8:kworker.0 00000100:00000001:0.0:1787828994.919511:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787828994.919513:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787828994.919514:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787828994.919515:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1787828994.919517:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828994.919520:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787828994.919521:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417875968 : -126234291675648 : ffff8d30c9e2de00) 00000100:00000001:0.0:1787828994.919523:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1787828994.919529:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:0.0:1787828994.919531:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1787828994.919533:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828994.919534:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828994.919535:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787828994.919537:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828994.919539:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298467875 00000100:00000001:0.0:1787828994.919540:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298469120 00000100:00000001:0.0:1787828994.919542:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298467875 00000100:00000001:0.0:1787828994.919544:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298469120 00000100:00000001:0.0:1787828994.919545:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787828994.919547:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828994.919548:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787828994.919550:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787828994/real 1787828994] req@ffff8d30c9e2de00 x1874670646785792/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829019 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787828994.919555:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787828994.919555:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787828994.919557:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787828994.919557:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787828994.919558:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828994.919559:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828994.919560:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787828994.919562:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2de00 x1874670646785792/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829019 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787828994.919567:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828994.919568:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787828994.919570:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828994.919571:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787828994.919572:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828994.919573:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787828994.919574:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787828994.919575:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787828994.919577:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787828994.919578:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787828994.919580:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646785792:192.168.203.143@tcp:8:kworker.0 00000100:00000001:0.0:1787828994.919582:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828994.919583:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828994.919584:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787828994.919585:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787828994.919587:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787828994.919588:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787828994.919589:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787828994.919590:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787828994.919591:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787828994.919595:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787828994.919596:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828994.919597:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828994.919598:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787828994.919599:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787828994.919618:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787828994.919619:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787828994.919619:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787828994.919621:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787828994.919622:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787828994.919624:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787828994.919625:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828994.919626:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787828994.919627:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787828994.919628:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787828994.919629:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787828994.919630:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787828994.919631:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787828994.919632:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787828994.919632:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787828994.919635:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787828994.919635:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828994.919636:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787828994.919638:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787828994.919639:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787828994.919640:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787828994.919641:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787828995.814680:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828995.814686:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828995.814747:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828995.814752:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828995.814755:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828996.838714:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828996.838722:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828996.838786:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828996.838855:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828996.838859:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828997.862679:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828997.862686:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828997.862734:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828997.862737:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828997.862740:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828998.886689:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828998.886695:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828998.886742:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828998.886747:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828998.886750:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787828999.910678:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787828999.910683:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787828999.910731:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787828999.910734:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787828999.910738:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787829000.038694:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829000.038703:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1787829000.038706:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787829000.038710:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787829000.038714:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787829000.038722:0:468:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 3800 00000100:00080000:3.0:1787829000.038752:0:468:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787829000.038756:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829000.038762:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:3.0:1787829000.038767:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1787829000.038770:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1787829000.038773:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:3.0:1787829000.038777:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:3.0:1787829000.038779:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829000.038781:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:3.0:1787829000.038786:0:468:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:3.0:1787829000.038789:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829000.038793:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787829000.038797:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829000.038801:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829000.038804:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:3.0:1787829000.038807:0:468:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829000.038818:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787829000.038820:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829000.038823:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787829000.038826:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787829000.038830:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829000.038840:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829000.038849:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c7016580 x1874670646785920/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787829000.038860:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787829000.038864:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829000.038877:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787829000.038881:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829000.038884:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1787829000.038886:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787829000.038888:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787829000.038891:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787829000.038894:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3800 00000100:00080000:3.0:1787829000.038898:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787829000.038901:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829000.038903:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:3.0:1787829000.038906:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1787829000.038907:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:0.0:1787829000.038910:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:3.0:1787829000.038910:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000020:00000001:3.0:1787829000.038915:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:0.0:1787829000.038916:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787829000.038918:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829000.038920:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787829000.038922:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:0.0:1787829000.038923:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787829000.038926:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787829000.038928:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 02000000:00000001:0.0:1787829000.038930:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:3.0:1787829000.038933:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829000.038934:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829000.038939:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787829000.038940:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829000.038947:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00100000:0.0:1787829000.038948:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c7016580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646785920:192.168.203.143@tcp:250:kworker.0 02000000:00000001:3.0:1787829000.038954:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829000.038957:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:3.0:1787829000.038958:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 02000000:00000001:0.0:1787829000.038964:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000008:00100000:3.0:1787829000.038965:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 02000000:00000001:0.0:1787829000.038970:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:3.0:1787829000.038970:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829000.038977:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:3.0:1787829000.038977:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829000.038982:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787829000.038984:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829000.038985:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829000.038990:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787829000.038991:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787829000.038993:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829000.039001:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829000.039002:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787829000.039005:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369545088 : -126234340006528 : ffff8d30c7016580) 00000100:00000001:0.0:1787829000.039010:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00100000:3.0:1787829000.039010:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c70161c0 x1874670646786048/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787829000.039022:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787829000.039025:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829000.039029:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787829000.039032:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829000.039036:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1787829000.039038:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787829000.039040:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787829000.039043:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787829000.039045:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3800 00000100:00080000:3.0:1787829000.039051:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1787829000.039053:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:3.0:1787829000.039054:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829000.039057:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:3.0:1787829000.039060:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1787829000.039062:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:0.0:1787829000.039064:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000020:00000001:3.0:1787829000.039065:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:0.0:1787829000.039069:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000020:00000001:3.0:1787829000.039070:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:0.0:1787829000.039072:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829000.039076:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1787829000.039077:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829000.039080:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829000.039081:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:0.0:1787829000.039085:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4298474240 00000100:00080000:3.0:1787829000.039089:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787829000.039094:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 00000100:00000001:3.0:1787829000.039094:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829000.039114:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:0.0:1787829000.039117:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4298474240 02000000:00000001:3.0:1787829000.039118:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829000.039123:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829000.039124:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 00000100:00000001:3.0:1787829000.039127:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1787829000.039129:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000008:00100000:3.0:1787829000.039131:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00000001:0.0:1787829000.039132:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:3.0:1787829000.039138:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829000.039139:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00000001:3.0:1787829000.039143:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787829000.039146:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829000/real 1787829000] req@ffff8d30c7016580 x1874670646785920/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829025 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1787829000.039150:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787829000.039154:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:0.0:1787829000.039156:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000001:3.0:1787829000.039158:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000400:00000001:0.0:1787829000.039159:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 02000000:00000001:3.0:1787829000.039162:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:0.0:1787829000.039164:0:2390:0:(events.c:78:reply_in_callback()) Process entered 02000000:00000001:3.0:1787829000.039166:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829000.039167:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787829000.039171:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829000.039173:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787829000.039175:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829000.039176:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787829000.039179:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c7016580 x1874670646785920/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829025 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787829000.039185:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c7015a40 x1874670646786176/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787829000.039210:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787829000.039214:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829000.039216:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829000.039219:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829000.039221:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829000.039224:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1787829000.039227:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829000.039233:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00080000:3.0:1787829000.039233:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1787829000.039236:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829000.039237:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:0.0:1787829000.039239:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787829000.039242:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829000.039242:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829000.039247:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00000001:3.0:1787829000.039249:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787829000.039252:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00080000:3.0:1787829000.039255:0:468:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3800 00000100:00000001:0.0:1787829000.039256:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787829000.039263:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c7016580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646785920:192.168.203.143@tcp:250:kworker.0 00000100:00080000:3.0:1787829000.039263:0:468:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787829000.039269:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829000.039271:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829000.039273:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:0.0:1787829000.039275:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000020:00000001:3.0:1787829000.039277:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:0.0:1787829000.039278:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 00000020:00000001:3.0:1787829000.039282:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 02000000:00000001:0.0:1787829000.039283:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 00000020:00000001:3.0:1787829000.039287:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 02000000:00000001:0.0:1787829000.039288:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787829000.039291:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787829000.039293:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 00000020:00000001:3.0:1787829000.039293:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:3.0:1787829000.039298:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 02000000:00000001:0.0:1787829000.039299:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 00000100:00000001:3.0:1787829000.039301:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 02000000:00000001:0.0:1787829000.039302:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787829000.039308:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00080000:3.0:1787829000.039309:0:468:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787829000.039311:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829000.039316:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829000.039317:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:3.0:1787829000.039320:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787829000.039324:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829000.039326:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829000.039328:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787829000.039328:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1787829000.039332:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000008:00100000:3.0:1787829000.039333:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00000001:0.0:1787829000.039338:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000008:00000001:3.0:1787829000.039339:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829000.039342:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787829000.039343:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829000.039347:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 02000000:00000001:3.0:1787829000.039349:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:0.0:1787829000.039350:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787829000.039353:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829000.039353:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829000.039358:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:3.0:1787829000.039358:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787829000.039362:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829000.039362:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787829000.039367:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829000.039368:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787829000.039377:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c70161c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646786048:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829000.039377:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829000.039384:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00100000:3.0:1787829000.039386:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c7014f00 x1874670646786304/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:0.0:1787829000.039388:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787829000.039392:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829000.039397:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1787829000.039402:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829000.039403:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787829000.039407:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829000.039411:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829000.039413:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787829000.039415:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369544128 : -126234340007488 : ffff8d30c70161c0) 00000100:00000001:0.0:1787829000.039419:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1787829000.039432:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:0.0:1787829000.039436:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1787829000.039438:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787829000.039439:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829000.039440:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787829000.039442:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829000.039445:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298472982 00000100:00000001:0.0:1787829000.039447:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298474240 00000100:00000001:0.0:1787829000.039449:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298472982 00000100:00000001:0.0:1787829000.039450:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298474240 00000100:00000001:0.0:1787829000.039452:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787829000.039453:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829000.039454:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787829000.039457:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829000/real 1787829000] req@ffff8d30c70161c0 x1874670646786048/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829025 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787829000.039462:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787829000.039463:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787829000.039465:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787829000.039466:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787829000.039467:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829000.039468:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787829000.039469:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787829000.039472:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c70161c0 x1874670646786048/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829025 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787829000.039481:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829000.039482:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787829000.039484:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829000.039485:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787829000.039486:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829000.039487:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787829000.039488:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787829000.039490:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829000.039492:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787829000.039494:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787829000.039497:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c70161c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646786048:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787829000.039500:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1787829000.039501:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787829000.039503:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829000.039505:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787829000.039507:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787829000.039509:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787829000.039512:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c7015a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646786176:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787829000.039515:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787829000.039517:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787829000.039519:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829000.039520:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1787829000.039523:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829000.039529:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787829000.039530:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369542208 : -126234340009408 : ffff8d30c7015a40) 00000100:00000001:0.0:1787829000.039535:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1787829000.039547:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:0.0:1787829000.039550:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1787829000.039552:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787829000.039553:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829000.039554:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787829000.039556:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829000.039559:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298472982 00000100:00000001:0.0:1787829000.039561:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298474240 00000100:00000001:0.0:1787829000.039562:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298472982 00000100:00000001:0.0:1787829000.039563:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298474240 00000100:00000001:0.0:1787829000.039565:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787829000.039566:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829000.039567:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787829000.039570:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829000/real 1787829000] req@ffff8d30c7015a40 x1874670646786176/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829025 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787829000.039575:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787829000.039575:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787829000.039577:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787829000.039578:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787829000.039579:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829000.039580:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787829000.039581:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787829000.039583:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c7015a40 x1874670646786176/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829025 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787829000.039588:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829000.039589:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787829000.039591:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829000.039592:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787829000.039593:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829000.039594:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787829000.039595:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787829000.039596:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829000.039598:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787829000.039623:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787829000.039628:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c7015a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646786176:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787829000.039632:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829000.039633:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787829000.039635:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787829000.039636:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787829000.039639:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787829000.039641:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787829000.039642:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787829000.039644:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787829000.039645:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787829000.039647:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787829000.039649:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829000.039651:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787829000.039652:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787829000.039653:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787829000.039655:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787829000.039656:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787829000.039657:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787829000.039659:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787829000.039660:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787829000.039662:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787829000.039663:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829000.039665:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829000.039668:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787829000.039669:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829000.039670:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787829000.039671:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787829000.039673:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1787829000.039674:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787829000.039675:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829000.039677:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787829000.039678:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787829000.039679:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787829000.039682:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c7014f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646786304:192.168.203.143@tcp:8:kworker.0 00000100:00000001:0.0:1787829000.039684:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787829000.039685:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787829000.039686:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829000.039688:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1787829000.039690:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829000.039694:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787829000.039696:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369539328 : -126234340012288 : ffff8d30c7014f00) 00000100:00000001:0.0:1787829000.039698:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1787829000.039707:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:0.0:1787829000.039710:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1787829000.039711:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787829000.039712:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829000.039713:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787829000.039715:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829000.039717:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298472996 00000100:00000001:0.0:1787829000.039719:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298474240 00000100:00000001:0.0:1787829000.039720:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298472996 00000100:00000001:0.0:1787829000.039722:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298474240 00000100:00000001:0.0:1787829000.039723:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787829000.039724:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829000.039726:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787829000.039728:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829000/real 1787829000] req@ffff8d30c7014f00 x1874670646786304/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829025 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787829000.039733:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787829000.039734:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787829000.039735:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787829000.039736:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787829000.039737:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829000.039738:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787829000.039739:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787829000.039741:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c7014f00 x1874670646786304/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829025 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787829000.039746:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829000.039748:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787829000.039750:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829000.039752:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787829000.039753:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829000.039755:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787829000.039757:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787829000.039760:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829000.039762:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787829000.039765:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787829000.039768:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c7014f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646786304:192.168.203.143@tcp:8:kworker.0 00000100:00000001:0.0:1787829000.039772:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829000.039774:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787829000.039775:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787829000.039777:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787829000.039779:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787829000.039780:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787829000.039782:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787829000.039784:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787829000.039785:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787829000.039795:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787829000.039797:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829000.039800:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829000.039802:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787829000.039803:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829000.039804:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787829000.039805:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829000.934659:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829000.934664:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829000.934704:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829000.934708:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829000.934710:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829001.958669:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829001.958674:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829001.958718:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829001.958722:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829001.958724:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829002.982679:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829002.982685:0:815:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1787829002.982687:0:815:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1787829002.982689:0:815:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829002.982693:0:815:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1787829002.982695:0:815:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829002.982696:0:815:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829002.982698:0:815:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829002.982754:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829002.982759:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829002.982762:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829004.006680:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829004.006687:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829004.006742:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829004.006754:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829004.006758:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829005.030699:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829005.030707:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829005.030770:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829005.030778:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829005.030792:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787829005.158787:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829005.158795:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1787829005.158798:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1787829005.158802:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787829005.158808:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1787829005.158815:0:40701:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 3805 00000100:00080000:1.0:1787829005.158826:0:40701:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1787829005.158830:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787829005.158833:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:1.0:1787829005.158837:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:1.0:1787829005.158840:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:1.0:1787829005.158844:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:1.0:1787829005.158848:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:1.0:1787829005.158851:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787829005.158852:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:1.0:1787829005.158858:0:40701:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:1.0:1787829005.158861:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829005.158865:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1787829005.158869:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829005.158879:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829005.158882:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:1.0:1787829005.158885:0:40701:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829005.158894:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1787829005.158896:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829005.158901:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1787829005.158904:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1787829005.158909:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829005.158922:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829005.158940:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d13ecb40 x1874670646786432/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787829005.158956:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787829005.158961:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829005.158976:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1787829005.158980:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829005.158983:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1787829005.158985:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1787829005.158987:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787829005.158991:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1787829005.158994:0:40701:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3805 00000100:00080000:1.0:1787829005.158999:0:40701:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787829005.159001:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787829005.159002:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787829005.159005:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:0.0:1787829005.159007:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000020:00000001:1.0:1787829005.159008:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:0.0:1787829005.159011:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000020:00000001:1.0:1787829005.159011:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:1.0:1787829005.159013:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:0.0:1787829005.159014:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000020:00000001:1.0:1787829005.159016:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:0.0:1787829005.159018:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829005.159019:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787829005.159023:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 02000000:00000001:0.0:1787829005.159024:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000100:00080000:1.0:1787829005.159026:0:40701:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 02000000:00000001:0.0:1787829005.159028:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829005.159029:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829005.159032:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829005.159033:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1787829005.159038:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829005.159040:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829005.159041:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:1.0:1787829005.159045:0:40701:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1787829005.159047:0:40701:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829005.159048:0:40701:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829005.159051:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1787829005.159053:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829005.159054:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1787829005.159055:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1787829005.159057:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829005.159060:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829005.159077:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d13ec000 x1874670646786560/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787829005.159087:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00100000:0.0:1787829005.159091:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d13ecb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646786432:192.168.203.143@tcp:250:kworker.0 00000100:00000001:1.0:1787829005.159091:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829005.159096:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829005.159101:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00080000:1.0:1787829005.159101:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829005.159106:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:1.0:1787829005.159108:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 02000000:00000001:0.0:1787829005.159109:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000100:00080000:1.0:1787829005.159109:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 02000000:00000001:0.0:1787829005.159114:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829005.159115:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1787829005.159119:0:40701:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3805 02000000:00000001:0.0:1787829005.159123:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00080000:1.0:1787829005.159123:0:40701:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1787829005.159127:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787829005.159128:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:1.0:1787829005.159129:0:40701:0:(genops.c:792:class_conn2export()) Process entered 02000000:00000001:0.0:1787829005.159130:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829005.159131:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:1.0:1787829005.159133:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000020:00000001:1.0:1787829005.159136:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:1.0:1787829005.159137:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787829005.159138:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:1.0:1787829005.159140:0:40701:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:1.0:1787829005.159143:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829005.159144:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1787829005.159146:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829005.159147:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829005.159149:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787829005.159150:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1787829005.159153:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839541340992 : -126234168210624 : ffff8d30d13ecb40) 00000008:00100000:1.0:1787829005.159154:0:40701:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1787829005.159159:0:40701:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829005.159161:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:1.0:1787829005.159162:0:40701:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829005.159168:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1787829005.159169:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829005.159170:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1787829005.159171:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1787829005.159172:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829005.159177:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829005.159180:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d13ec780 x1874670646786688/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787829005.159186:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787829005.159187:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829005.159189:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1787829005.159192:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829005.159195:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000800:00000100:0.0:1787829005.159199:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787829005.159199:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1787829005.159203:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787829005.159206:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1787829005.159208:0:40701:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3805 00000100:00080000:1.0:1787829005.159213:0:40701:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787829005.159214:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787829005.159215:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787829005.159218:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:0.0:1787829005.159220:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000020:00000001:1.0:1787829005.159221:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:0.0:1787829005.159225:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829005.159225:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:1.0:1787829005.159228:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:0.0:1787829005.159229:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000020:00000001:1.0:1787829005.159233:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:1.0:1787829005.159235:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829005.159236:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829005.159237:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:0.0:1787829005.159240:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4298479360 00000100:00080000:1.0:1787829005.159243:0:40701:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:1.0:1787829005.159248:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829005.159249:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1787829005.159251:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787829005.159252:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 02000000:00000001:1.0:1787829005.159253:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829005.159257:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:1.0:1787829005.159260:0:40701:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000008:00000001:1.0:1787829005.159262:0:40701:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829005.159263:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4298479360 00000100:00000001:1.0:1787829005.159264:0:40701:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829005.159271:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 02000000:00000001:1.0:1787829005.159272:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1787829005.159274:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829005.159277:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1787829005.159279:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:0.0:1787829005.159280:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829005.159282:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829005.159283:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829005.159287:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00000001:1.0:1787829005.159287:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787829005.159297:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829005/real 1787829005] req@ffff8d30d13ecb40 x1874670646786432/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829030 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829005.159298:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d13ec3c0 x1874670646786816/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000001:0.0:1787829005.159322:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000001:1.0:1787829005.159323:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000400:00000001:0.0:1787829005.159326:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787829005.159327:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829005.159333:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787829005.159334:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829005.159337:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787829005.159340:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829005.159345:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787829005.159347:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787829005.159353:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d13ecb40 x1874670646786432/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829030 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787829005.159360:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829005.159366:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787829005.159382:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829005.159385:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787829005.159388:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829005.159391:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787829005.159395:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787829005.159398:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829005.159403:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787829005.159405:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787829005.159408:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d13ecb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646786432:192.168.203.143@tcp:250:kworker.0 00000100:00000001:0.0:1787829005.159412:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829005.159414:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787829005.159416:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787829005.159418:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787829005.159421:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787829005.159423:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787829005.159425:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787829005.159428:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787829005.159430:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787829005.159432:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787829005.159433:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829005.159436:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829005.159442:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787829005.159444:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829005.159446:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787829005.159448:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787829005.159449:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1787829005.159450:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787829005.159451:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829005.159453:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787829005.159455:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787829005.159456:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787829005.159459:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d13ec000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646786560:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787829005.159462:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787829005.159464:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787829005.159465:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829005.159467:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1787829005.159469:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829005.159474:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787829005.159475:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839541338112 : -126234168213504 : ffff8d30d13ec000) 00000100:00000001:0.0:1787829005.159479:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1787829005.159487:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:0.0:1787829005.159490:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1787829005.159492:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787829005.159493:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829005.159494:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787829005.159496:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829005.159499:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298478102 00000100:00000001:0.0:1787829005.159501:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298479360 00000100:00000001:0.0:1787829005.159502:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298478102 00000100:00000001:0.0:1787829005.159504:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298479360 00000100:00000001:0.0:1787829005.159505:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787829005.159506:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829005.159508:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787829005.159517:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829005/real 1787829005] req@ffff8d30d13ec000 x1874670646786560/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829030 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787829005.159522:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787829005.159523:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787829005.159524:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787829005.159525:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787829005.159526:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829005.159528:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787829005.159529:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787829005.159531:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d13ec000 x1874670646786560/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829030 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787829005.159536:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829005.159537:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787829005.159539:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829005.159539:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787829005.159540:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829005.159542:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787829005.159543:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787829005.159545:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829005.159547:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787829005.159548:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787829005.159550:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d13ec000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646786560:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787829005.159553:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1787829005.159553:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787829005.159554:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829005.159556:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787829005.159557:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787829005.159558:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787829005.159560:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d13ec780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646786688:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787829005.159562:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787829005.159563:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787829005.159564:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829005.159565:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1787829005.159567:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829005.159574:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787829005.159575:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839541340032 : -126234168211584 : ffff8d30d13ec780) 00000100:00000001:0.0:1787829005.159578:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1787829005.159584:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:0.0:1787829005.159587:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1787829005.159588:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787829005.159589:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829005.159590:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787829005.159592:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829005.159594:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298478102 00000100:00000001:0.0:1787829005.159595:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298479360 00000100:00000001:0.0:1787829005.159597:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298478102 00000100:00000001:0.0:1787829005.159598:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298479360 00000100:00000001:0.0:1787829005.159638:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787829005.159640:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829005.159642:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787829005.159645:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829005/real 1787829005] req@ffff8d30d13ec780 x1874670646786688/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829030 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787829005.159652:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787829005.159653:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787829005.159656:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787829005.159658:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787829005.159659:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829005.159661:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787829005.159662:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787829005.159665:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d13ec780 x1874670646786688/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829030 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787829005.159669:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829005.159670:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787829005.159672:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829005.159673:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787829005.159674:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829005.159675:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787829005.159676:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787829005.159677:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829005.159680:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787829005.159681:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787829005.159683:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d13ec780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646786688:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787829005.159725:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1787829005.159728:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787829005.159729:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829005.159732:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787829005.159734:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787829005.159736:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787829005.159740:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d13ec3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646786816:192.168.203.143@tcp:8:kworker.0 00000100:00000001:0.0:1787829005.159745:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787829005.159747:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787829005.159750:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829005.159753:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1787829005.159756:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829005.159762:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787829005.159764:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839541339072 : -126234168212544 : ffff8d30d13ec3c0) 00000100:00000001:0.0:1787829005.159768:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1787829005.159778:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:0.0:1787829005.159781:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1787829005.159783:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787829005.159784:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829005.159785:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787829005.159787:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829005.159789:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298478116 00000100:00000001:0.0:1787829005.159791:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298479360 00000100:00000001:0.0:1787829005.159793:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298478116 00000100:00000001:0.0:1787829005.159794:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298479360 00000100:00000001:0.0:1787829005.159796:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787829005.159797:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829005.159798:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787829005.159801:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829005/real 1787829005] req@ffff8d30d13ec3c0 x1874670646786816/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829030 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787829005.159805:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787829005.159806:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787829005.159808:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787829005.159809:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787829005.159810:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829005.159811:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787829005.159812:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787829005.159814:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d13ec3c0 x1874670646786816/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829030 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787829005.159823:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829005.159824:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787829005.159826:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829005.159827:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787829005.159828:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829005.159829:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787829005.159830:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787829005.159831:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829005.159833:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787829005.159834:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787829005.159836:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d13ec3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646786816:192.168.203.143@tcp:8:kworker.0 00000100:00000001:0.0:1787829005.159838:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829005.159839:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787829005.159840:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787829005.159841:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787829005.159842:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787829005.159843:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787829005.159844:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787829005.159845:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787829005.159846:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787829005.159847:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787829005.159848:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829005.159849:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787829005.159849:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787829005.159850:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787829005.159851:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787829005.159852:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787829005.159853:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787829005.159854:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787829005.159855:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787829005.159856:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787829005.159857:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829005.159857:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787829005.159858:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787829005.159859:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787829005.159861:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787829005.159862:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787829005.159862:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787829005.159864:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787829005.159865:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787829005.159866:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787829005.159867:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829005.159868:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829005.159870:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787829005.159871:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829005.159872:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787829005.159873:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829006.054693:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829006.054702:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829006.054762:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829006.054766:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829006.054769:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829007.078704:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829007.078710:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829007.078771:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829007.078775:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829007.078778:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829008.102699:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829008.102707:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829008.102756:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829008.102761:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829008.102765:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829009.126752:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829009.126769:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829009.126841:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829009.126846:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829009.126850:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829010.150671:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829010.150676:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829010.150716:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829010.150719:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829010.150721:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787829010.278721:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829010.278728:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1787829010.278731:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787829010.278733:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787829010.278737:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787829010.278744:0:468:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 3810 00000100:00080000:3.0:1787829010.278755:0:468:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787829010.278758:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829010.278760:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:3.0:1787829010.278765:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1787829010.278775:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1787829010.278778:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:3.0:1787829010.278781:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:3.0:1787829010.278784:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829010.278786:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:3.0:1787829010.278790:0:468:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:3.0:1787829010.278793:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829010.278794:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787829010.278797:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829010.278799:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829010.278801:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:3.0:1787829010.278803:0:468:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829010.278813:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787829010.278814:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829010.278817:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787829010.278819:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787829010.278822:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829010.278830:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829010.278838:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c70143c0 x1874670646786944/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787829010.278846:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787829010.278849:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829010.278860:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787829010.278862:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829010.278865:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1787829010.278866:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787829010.278867:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787829010.278869:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787829010.278871:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3810 00000100:00080000:3.0:1787829010.278873:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787829010.278875:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829010.278876:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:3.0:1787829010.278878:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1787829010.278879:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1787829010.278880:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000020:00000001:3.0:1787829010.278882:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:3.0:1787829010.278884:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829010.278885:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:3.0:1787829010.278887:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:3.0:1787829010.278888:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829010.278889:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787829010.278894:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829010.278895:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829010.278896:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:3.0:1787829010.278899:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1787829010.278901:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829010.278903:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829010.278905:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787829010.278906:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829010.278908:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787829010.278908:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787829010.278910:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829010.278912:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829010.278916:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c7015680 x1874670646787072/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787829010.278921:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787829010.278923:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829010.278924:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787829010.278927:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829010.278929:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1787829010.278931:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787829010.278931:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787829010.278933:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787829010.278935:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3810 00000100:00080000:3.0:1787829010.278937:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787829010.278939:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829010.278942:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:3.0:1787829010.278944:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1787829010.278946:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1787829010.278948:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000020:00000001:3.0:1787829010.278950:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:3.0:1787829010.278951:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829010.278952:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:3.0:1787829010.278954:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:3.0:1787829010.278956:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829010.278957:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787829010.278958:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829010.278959:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829010.278961:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:3.0:1787829010.278962:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1787829010.278965:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829010.278966:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1787829010.278966:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829010.278974:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:0.0:1787829010.278976:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 02000000:00000001:3.0:1787829010.278976:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829010.278982:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:0.0:1787829010.278983:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 02000000:00000001:3.0:1787829010.278983:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787829010.278991:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829010.278992:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787829010.279000:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829010.279002:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829010.279010:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000100:00100000:3.0:1787829010.279011:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c70152c0 x1874670646787200/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:0.0:1787829010.279015:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787829010.279024:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829010.279048:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00100000:0.0:1787829010.279053:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c70143c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646786944:192.168.203.143@tcp:250:kworker.0 00000100:00000001:3.0:1787829010.279053:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829010.279061:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829010.279063:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00080000:3.0:1787829010.279066:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829010.279069:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 02000000:00000001:0.0:1787829010.279070:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:3.0:1787829010.279071:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 02000000:00000001:0.0:1787829010.279077:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787829010.279077:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 02000000:00000001:0.0:1787829010.279084:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:3.0:1787829010.279084:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787829010.279092:0:468:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3810 00000100:00080000:3.0:1787829010.279095:0:468:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 02000000:00000001:0.0:1787829010.279097:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829010.279099:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829010.279103:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:3.0:1787829010.279107:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1787829010.279108:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1787829010.279111:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000020:00000001:3.0:1787829010.279113:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:3.0:1787829010.279114:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829010.279115:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:3.0:1787829010.279119:0:468:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:3.0:1787829010.279121:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829010.279122:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 02000000:00000001:3.0:1787829010.279123:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:0.0:1787829010.279129:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369536448 : -126234340015168 : ffff8d30c70143c0) 02000000:00000001:3.0:1787829010.279130:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829010.279142:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829010.279143:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:3.0:1787829010.279149:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:3.0:1787829010.279151:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1787829010.279153:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829010.279155:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829010.279158:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787829010.279159:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829010.279161:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787829010.279162:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787829010.279164:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829010.279168:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829010.279172:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c7014b40 x1874670646787328/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787829010.279178:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787829010.279180:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829010.279181:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:0.0:1787829010.279193:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:0.0:1787829010.279208:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1787829010.279213:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787829010.279215:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829010.279216:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787829010.279221:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829010.279225:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4298484480 00000100:00000001:0.0:1787829010.279235:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1787829010.279243:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4298484480 00000100:00000001:0.0:1787829010.279246:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1787829010.279248:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787829010.279251:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829010.279253:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787829010.279259:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829010/real 1787829010] req@ffff8d30c70143c0 x1874670646786944/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829035 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787829010.279267:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787829010.279269:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787829010.279271:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787829010.279274:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787829010.279275:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829010.279277:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787829010.279279:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787829010.279283:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c70143c0 x1874670646786944/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829035 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787829010.279288:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829010.279291:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787829010.279293:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829010.279294:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787829010.279295:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829010.279297:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787829010.279298:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787829010.279301:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829010.279304:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787829010.279306:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787829010.279309:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c70143c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646786944:192.168.203.143@tcp:250:kworker.0 00000100:00000001:0.0:1787829010.279312:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1787829010.279313:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787829010.279314:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829010.279316:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787829010.279318:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787829010.279320:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787829010.279322:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c7015680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646787072:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787829010.279325:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787829010.279326:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787829010.279327:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829010.279328:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1787829010.279330:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829010.279334:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787829010.279335:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369541248 : -126234340010368 : ffff8d30c7015680) 00000100:00000001:0.0:1787829010.279339:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1787829010.279345:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:0.0:1787829010.279348:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1787829010.279349:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787829010.279350:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829010.279352:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787829010.279353:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829010.279357:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298483222 00000100:00000001:0.0:1787829010.279359:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298484480 00000100:00000001:0.0:1787829010.279361:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298483222 00000100:00000001:0.0:1787829010.279362:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298484480 00000100:00000001:0.0:1787829010.279364:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787829010.279365:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829010.279366:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787829010.279368:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829010/real 1787829010] req@ffff8d30c7015680 x1874670646787072/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829035 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787829010.279373:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787829010.279374:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787829010.279375:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787829010.279376:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787829010.279377:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829010.279378:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787829010.279379:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787829010.279381:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c7015680 x1874670646787072/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829035 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787829010.279387:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829010.279388:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787829010.279389:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829010.279390:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787829010.279391:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829010.279392:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787829010.279393:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787829010.279395:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829010.279397:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787829010.279398:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787829010.279400:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c7015680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646787072:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787829010.279402:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829010.279404:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787829010.279405:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787829010.279407:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787829010.279410:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787829010.279412:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787829010.279413:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787829010.279418:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787829010.279419:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787829010.279421:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787829010.279422:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829010.279423:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787829010.279423:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787829010.279424:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787829010.279425:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787829010.279426:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787829010.279427:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787829010.279428:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787829010.279429:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787829010.279430:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787829010.279431:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829010.279433:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829010.279441:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787829010.279442:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829010.279444:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787829010.279445:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787829010.279446:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1787829010.279448:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787829010.279449:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829010.279450:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787829010.279451:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787829010.279452:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787829010.279454:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c70152c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646787200:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787829010.279456:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787829010.279458:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787829010.279459:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829010.279460:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1787829010.279462:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829010.279465:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787829010.279466:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369540288 : -126234340011328 : ffff8d30c70152c0) 00000100:00000001:0.0:1787829010.279469:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1787829010.279475:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:0.0:1787829010.279477:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1787829010.279478:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787829010.279480:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829010.279481:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787829010.279482:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829010.279485:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298483222 00000100:00000001:0.0:1787829010.279487:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298484480 00000100:00000001:0.0:1787829010.279490:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298483222 00000100:00000001:0.0:1787829010.279492:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298484480 00000100:00000001:0.0:1787829010.279495:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787829010.279496:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829010.279498:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787829010.279502:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829010/real 1787829010] req@ffff8d30c70152c0 x1874670646787200/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829035 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787829010.279518:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787829010.279519:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787829010.279522:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787829010.279524:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787829010.279526:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829010.279528:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787829010.279529:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787829010.279533:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c70152c0 x1874670646787200/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829035 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787829010.279541:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829010.279544:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787829010.279546:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829010.279548:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787829010.279549:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829010.279551:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787829010.279552:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787829010.279554:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829010.279557:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787829010.279559:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787829010.279561:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c70152c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646787200:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787829010.279564:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1787829010.279565:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787829010.279567:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829010.279568:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787829010.279570:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787829010.279571:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787829010.279573:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c7014b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646787328:192.168.203.143@tcp:8:kworker.0 00000100:00000001:0.0:1787829010.279575:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787829010.279577:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787829010.279578:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829010.279579:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1787829010.279582:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829010.279586:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787829010.279587:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369538368 : -126234340013248 : ffff8d30c7014b40) 00000100:00000001:0.0:1787829010.279590:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1787829010.279599:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:0.0:1787829010.279653:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1787829010.279654:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787829010.279656:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829010.279657:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787829010.279659:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829010.279661:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298483236 00000100:00000001:0.0:1787829010.279694:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298484480 00000100:00000001:0.0:1787829010.279697:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298483236 00000100:00000001:0.0:1787829010.279698:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298484480 00000100:00000001:0.0:1787829010.279701:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787829010.279702:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829010.279704:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787829010.279710:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829010/real 1787829010] req@ffff8d30c7014b40 x1874670646787328/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829035 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787829010.279715:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787829010.279717:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787829010.279719:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787829010.279720:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787829010.279721:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829010.279723:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787829010.279724:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787829010.279729:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c7014b40 x1874670646787328/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829035 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787829010.279734:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829010.279738:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787829010.279741:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829010.279743:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787829010.279744:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829010.279745:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787829010.279746:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787829010.279748:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829010.279750:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787829010.279751:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787829010.279753:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c7014b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646787328:192.168.203.143@tcp:8:kworker.0 00000100:00000001:0.0:1787829010.279755:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829010.279756:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787829010.279757:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787829010.279758:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787829010.279759:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787829010.279760:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787829010.279761:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787829010.279763:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787829010.279764:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787829010.279765:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787829010.279766:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829010.279767:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787829010.279768:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787829010.279769:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787829010.279770:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787829010.279771:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787829010.279772:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787829010.279773:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787829010.279773:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787829010.279774:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787829010.279775:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829010.279776:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829010.279778:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787829010.279779:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829010.279780:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787829010.279781:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829011.174725:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829011.174736:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829011.174803:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829011.174809:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829011.174860:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829012.198751:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829012.198760:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829012.198839:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829012.198844:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829012.198848:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829013.222712:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829013.222720:0:815:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1787829013.222723:0:815:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1787829013.222742:0:815:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829013.222749:0:815:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1787829013.222753:0:815:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829013.222756:0:815:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829013.222759:0:815:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829013.222873:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829013.222879:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829013.222884:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829014.246734:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829014.246742:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829014.246888:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829014.246894:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829014.246910:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829015.270732:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829015.270741:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829015.270822:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829015.270827:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829015.270831:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787829015.398717:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829015.398730:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1787829015.398732:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787829015.398736:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787829015.398741:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787829015.398748:0:40798:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 3815 00000100:00080000:0.0:1787829015.398758:0:40798:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787829015.398761:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829015.398763:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787829015.398768:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787829015.398770:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787829015.398774:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:0.0:1787829015.398776:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:0.0:1787829015.398778:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829015.398780:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:0.0:1787829015.398784:0:40798:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787829015.398786:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829015.398789:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787829015.398792:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787829015.398794:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829015.398797:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1787829015.398799:0:40798:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829015.398831:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787829015.398833:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829015.398835:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787829015.398837:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787829015.398842:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829015.398850:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787829015.398857:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d27c00 x1874670646787456/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787829015.398865:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787829015.398868:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829015.398900:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787829015.398904:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829015.398907:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1787829015.398909:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787829015.398910:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787829015.398913:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787829015.398916:0:40798:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3815 00000100:00080000:0.0:1787829015.398920:0:40798:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787829015.398921:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829015.398923:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787829015.398927:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787829015.398928:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787829015.398930:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000020:00000001:0.0:1787829015.398933:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:0.0:1787829015.398935:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829015.398936:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:0.0:1787829015.398941:0:40798:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787829015.398941:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787829015.398947:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829015.398949:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 02000000:00000001:0.0:1787829015.398950:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:2.0:1787829015.398954:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 02000000:00000001:0.0:1787829015.398959:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787829015.398959:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787829015.398961:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829015.398964:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829015.398967:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 02000000:00000001:2.0:1787829015.398968:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000008:00100000:0.0:1787829015.398972:0:40798:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 02000000:00000001:2.0:1787829015.398973:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000008:00000001:0.0:1787829015.398980:0:40798:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829015.398981:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829015.398985:0:40798:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829015.399029:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00100000:2.0:1787829015.399029:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d27c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646787456:192.168.203.143@tcp:250:kworker.0 02000000:00000001:0.0:1787829015.399031:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829015.399035:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:2.0:1787829015.399037:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787829015.399038:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787829015.399041:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829015.399042:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1787829015.399046:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829015.399048:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829015.399048:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1787829015.399055:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787829015.399056:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d252c0 x1874670646787584/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787829015.399067:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787829015.399069:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787829015.399070:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829015.399072:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601351680 : -126234108199936 : ffff8d30d4d27c00) 00000100:00000001:0.0:1787829015.399075:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829015.399077:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00080000:0.0:1787829015.399079:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829015.399082:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1787829015.399083:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787829015.399084:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787829015.399087:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787829015.399089:0:40798:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3815 00000100:00080000:0.0:1787829015.399092:0:40798:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787829015.399093:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829015.399094:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787829015.399096:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787829015.399096:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787829015.399098:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000020:00000001:0.0:1787829015.399100:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:0.0:1787829015.399102:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829015.399103:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:0.0:1787829015.399105:0:40798:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787829015.399107:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829015.399108:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787829015.399111:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000800:00000100:2.0:1787829015.399112:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 02000000:00000001:0.0:1787829015.399114:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829015.399118:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1787829015.399120:0:40798:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1787829015.399122:0:40798:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829015.399123:0:40798:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829015.399127:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787829015.399128:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829015.399128:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1787829015.399132:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787829015.399134:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:2.0:1787829015.399135:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 02000000:00000001:0.0:1787829015.399136:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829015.399138:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829015.399140:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829015.399141:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00100000:0.0:1787829015.399148:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d25680 x1874670646787712/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787829015.399154:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787829015.399154:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829015.399156:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829015.399160:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829015.399160:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4298489600 00000100:00080000:0.0:1787829015.399162:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829015.399166:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1787829015.399167:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787829015.399168:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787829015.399170:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:2.0:1787829015.399171:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 00000100:00080000:0.0:1787829015.399172:0:40798:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3815 00000100:00080000:0.0:1787829015.399177:0:40798:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787829015.399178:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4298489600 00000100:00000001:0.0:1787829015.399180:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829015.399182:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787829015.399186:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:2.0:1787829015.399186:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 00000020:00000001:0.0:1787829015.399189:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:2.0:1787829015.399191:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787829015.399194:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829015.399196:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000020:00000001:0.0:1787829015.399198:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00100000:2.0:1787829015.399201:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829015/real 1787829015] req@ffff8d30d4d27c00 x1874670646787456/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829040 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:0.0:1787829015.399202:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:0.0:1787829015.399206:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829015.399209:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:0.0:1787829015.399216:0:40798:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787829015.399220:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:2.0:1787829015.399221:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 02000000:00000001:0.0:1787829015.399222:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000400:00000001:2.0:1787829015.399224:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 02000000:00000001:0.0:1787829015.399226:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787829015.399228:0:2390:0:(events.c:78:reply_in_callback()) Process entered 02000000:00000001:0.0:1787829015.399229:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829015.399232:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:2.0:1787829015.399233:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000008:00100000:0.0:1787829015.399235:0:40798:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00000001:2.0:1787829015.399236:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000008:00000001:0.0:1787829015.399241:0:40798:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829015.399242:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787829015.399245:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829015.399246:0:40798:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829015.399252:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00100000:2.0:1787829015.399252:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d27c00 x1874670646787456/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829040 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:0.0:1787829015.399254:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829015.399258:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787829015.399260:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787829015.399263:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829015.399269:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829015.399269:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829015.399273:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00100000:0.0:1787829015.399276:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d26940 x1874670646787840/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787829015.399277:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829015.399281:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787829015.399283:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829015.399287:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787829015.399290:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787829015.399291:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00080000:2.0:1787829015.399293:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00000001:0.0:1787829015.399294:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829015.399298:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787829015.399299:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:2.0:1787829015.399303:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1787829015.399306:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d27c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646787456:192.168.203.143@tcp:250:kworker.0 00000100:00000001:2.0:1787829015.399310:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829015.399311:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787829015.399312:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787829015.399314:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1787829015.399317:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787829015.399318:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:2.0:1787829015.399320:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:2.0:1787829015.399323:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787829015.399324:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1787829015.399327:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787829015.399328:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829015.399329:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829015.399335:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1787829015.399337:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829015.399338:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787829015.399339:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1787829015.399340:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1787829015.399341:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787829015.399343:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829015.399345:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787829015.399347:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787829015.399349:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787829015.399352:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d252c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646787584:192.168.203.143@tcp:38:kworker.0 00000100:00000001:2.0:1787829015.399356:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787829015.399358:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1787829015.399360:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829015.399361:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1787829015.399364:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829015.399370:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787829015.399372:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601341120 : -126234108210496 : ffff8d30d4d252c0) 00000100:00000001:2.0:1787829015.399377:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1787829015.399390:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:2.0:1787829015.399395:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1787829015.399398:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787829015.399399:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829015.399401:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1787829015.399404:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829015.399408:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298488342 00000100:00000001:2.0:1787829015.399410:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298489600 00000100:00000001:2.0:1787829015.399411:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298488342 00000100:00000001:2.0:1787829015.399413:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298489600 00000100:00000001:2.0:1787829015.399414:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787829015.399416:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829015.399417:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1787829015.399420:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829015/real 1787829015] req@ffff8d30d4d252c0 x1874670646787584/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829040 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1787829015.399426:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787829015.399427:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1787829015.399429:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1787829015.399436:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787829015.399437:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829015.399439:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787829015.399440:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787829015.399442:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d252c0 x1874670646787584/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829040 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787829015.399447:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829015.399449:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787829015.399450:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829015.399451:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787829015.399452:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829015.399453:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787829015.399454:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787829015.399456:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829015.399458:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:2.0:1787829015.399460:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1787829015.399462:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d252c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646787584:192.168.203.143@tcp:38:kworker.0 00000100:00000001:2.0:1787829015.399465:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1787829015.399466:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787829015.399467:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829015.399468:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787829015.399469:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787829015.399470:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787829015.399472:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d25680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646787712:192.168.203.143@tcp:38:kworker.0 00000100:00000001:2.0:1787829015.399474:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787829015.399475:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1787829015.399477:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829015.399478:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1787829015.399480:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829015.399483:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787829015.399484:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601342080 : -126234108209536 : ffff8d30d4d25680) 00000100:00000001:2.0:1787829015.399487:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1787829015.399495:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:2.0:1787829015.399498:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1787829015.399499:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787829015.399500:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829015.399501:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1787829015.399503:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829015.399505:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298488342 00000100:00000001:2.0:1787829015.399507:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298489600 00000100:00000001:2.0:1787829015.399508:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298488342 00000100:00000001:2.0:1787829015.399510:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298489600 00000100:00000001:2.0:1787829015.399511:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787829015.399513:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829015.399514:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1787829015.399516:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829015/real 1787829015] req@ffff8d30d4d25680 x1874670646787712/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829040 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1787829015.399526:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787829015.399526:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1787829015.399528:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1787829015.399529:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787829015.399530:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829015.399531:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787829015.399532:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787829015.399534:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d25680 x1874670646787712/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829040 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787829015.399538:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829015.399540:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787829015.399541:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829015.399542:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787829015.399543:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829015.399544:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787829015.399545:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787829015.399547:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829015.399548:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:2.0:1787829015.399550:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1787829015.399551:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d25680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646787712:192.168.203.143@tcp:38:kworker.0 00000100:00000001:2.0:1787829015.399554:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1787829015.399554:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787829015.399555:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829015.399556:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787829015.399558:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787829015.399559:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787829015.399560:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d26940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646787840:192.168.203.143@tcp:8:kworker.0 00000100:00000001:2.0:1787829015.399562:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787829015.399563:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1787829015.399564:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829015.399566:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1787829015.399567:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829015.399570:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787829015.399571:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601346880 : -126234108204736 : ffff8d30d4d26940) 00000100:00000001:2.0:1787829015.399574:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1787829015.399579:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:2.0:1787829015.399581:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1787829015.399583:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787829015.399583:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829015.399584:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1787829015.399586:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829015.399588:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298488355 00000100:00000001:2.0:1787829015.399589:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298489600 00000100:00000001:2.0:1787829015.399591:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298488355 00000100:00000001:2.0:1787829015.399592:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298489600 00000100:00000001:2.0:1787829015.399594:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787829015.399595:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829015.399596:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1787829015.399629:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829015/real 1787829015] req@ffff8d30d4d26940 x1874670646787840/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829040 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1787829015.399637:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787829015.399638:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1787829015.399647:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1787829015.399649:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787829015.399650:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829015.399652:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787829015.399653:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787829015.399657:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d26940 x1874670646787840/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829040 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787829015.399664:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829015.399666:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787829015.399669:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829015.399671:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787829015.399672:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829015.399673:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787829015.399675:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787829015.399678:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829015.399680:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:2.0:1787829015.399682:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1787829015.399685:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d26940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646787840:192.168.203.143@tcp:8:kworker.0 00000100:00000001:2.0:1787829015.399689:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829015.399691:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787829015.399692:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787829015.399694:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1787829015.399696:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787829015.399698:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:2.0:1787829015.399699:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:2.0:1787829015.399701:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787829015.399703:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1787829015.399705:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787829015.399706:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829015.399708:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787829015.399709:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787829015.399710:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1787829015.399712:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787829015.399713:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:2.0:1787829015.399714:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:2.0:1787829015.399716:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787829015.399717:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1787829015.399718:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787829015.399719:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829015.399721:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787829015.399722:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787829015.399723:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1787829015.399724:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787829015.399726:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:2.0:1787829015.399727:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:2.0:1787829015.399728:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787829015.399729:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1787829015.399732:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787829015.399733:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829015.399735:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829015.399739:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1787829015.399740:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829015.399742:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787829015.399743:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829016.294701:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829016.294709:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829016.294767:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829016.294772:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829016.294775:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829017.318720:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829017.318727:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829017.318783:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829017.318786:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829017.318789:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829018.342731:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829018.342739:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829018.342813:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829018.342818:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829018.342822:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829019.366719:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829019.366727:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829019.366799:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829019.366803:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829019.366808:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829020.390717:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829020.390723:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829020.390835:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829020.390838:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829020.390841:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787829020.518747:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829020.518775:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1787829020.518778:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787829020.518782:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787829020.518788:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787829020.518796:0:40798:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 3821 00000100:00080000:0.0:1787829020.518807:0:40798:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787829020.518811:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829020.518814:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787829020.518818:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787829020.518820:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787829020.518823:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:0.0:1787829020.518840:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:0.0:1787829020.518842:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829020.518843:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:0.0:1787829020.518847:0:40798:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787829020.518849:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829020.518851:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787829020.518854:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787829020.518856:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829020.518858:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1787829020.518860:0:40798:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829020.518870:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787829020.518871:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829020.518874:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787829020.518877:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787829020.518880:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829020.518908:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787829020.518920:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d26940 x1874670646787968/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787829020.518933:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787829020.518938:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829020.519000:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787829020.519003:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829020.519006:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1787829020.519008:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787829020.519010:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787829020.519012:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:2.0:1787829020.519014:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00080000:0.0:1787829020.519015:0:40798:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3821 00000100:00000001:2.0:1787829020.519020:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00080000:0.0:1787829020.519021:0:40798:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787829020.519024:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1787829020.519025:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829020.519028:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:2.0:1787829020.519028:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787829020.519031:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787829020.519032:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787829020.519034:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787829020.519037:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000020:00000001:0.0:1787829020.519038:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:0.0:1787829020.519040:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829020.519041:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 02000000:00000001:2.0:1787829020.519044:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000100:00080000:0.0:1787829020.519045:0:40798:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787829020.519066:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829020.519066:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787829020.519073:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787829020.519073:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829020.519081:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00100000:2.0:1787829020.519081:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d26940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646787968:192.168.203.143@tcp:250:kworker.0 02000000:00000001:0.0:1787829020.519087:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829020.519089:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:0.0:1787829020.519090:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 02000000:00000001:2.0:1787829020.519094:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000008:00100000:0.0:1787829020.519096:0:40798:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 02000000:00000001:2.0:1787829020.519098:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:0.0:1787829020.519099:0:40798:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829020.519101:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:0.0:1787829020.519103:0:40798:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829020.519108:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829020.519109:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787829020.519112:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829020.519114:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787829020.519131:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787829020.519134:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829020.519142:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829020.519144:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787829020.519148:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601346880 : -126234108204736 : ffff8d30d4d26940) 00000100:00100000:0.0:1787829020.519154:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d25680 x1874670646788096/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787829020.519156:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:0.0:1787829020.519168:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787829020.519172:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829020.519177:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787829020.519180:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829020.519183:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1787829020.519185:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787829020.519188:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787829020.519192:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787829020.519195:0:40798:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3821 00000100:00080000:0.0:1787829020.519200:0:40798:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787829020.519203:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829020.519207:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000800:00000100:2.0:1787829020.519208:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000020:00000001:0.0:1787829020.519235:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787829020.519237:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787829020.519239:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000020:00000001:0.0:1787829020.519241:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:0.0:1787829020.519243:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829020.519245:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:2.0:1787829020.519247:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00080000:0.0:1787829020.519249:0:40798:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787829020.519253:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829020.519254:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 02000000:00000001:0.0:1787829020.519255:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:2.0:1787829020.519258:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829020.519259:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787829020.519261:0:2390:0:(events.c:68:request_out_callback()) Process leaving 02000000:00000001:0.0:1787829020.519262:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829020.519265:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1787829020.519267:0:40798:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00000001:2.0:1787829020.519268:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:0.0:1787829020.519269:0:40798:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829020.519273:0:40798:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829020.519274:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4298494720 02000000:00000001:0.0:1787829020.519277:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787829020.519279:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829020.519281:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787829020.519282:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787829020.519283:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829020.519284:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1787829020.519288:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829020.519290:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4298494720 00000100:00000001:2.0:1787829020.519297:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 00000100:00000001:2.0:1787829020.519299:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00100000:0.0:1787829020.519302:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d252c0 x1874670646788224/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787829020.519302:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829020.519307:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1787829020.519314:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829020/real 1787829020] req@ffff8d30d4d26940 x1874670646787968/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829045 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787829020.519324:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787829020.519330:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:2.0:1787829020.519330:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787829020.519333:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787829020.519338:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829020.519338:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1787829020.519342:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00080000:0.0:1787829020.519344:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:2.0:1787829020.519345:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829020.519349:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00080000:0.0:1787829020.519354:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1787829020.519354:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829020.519359:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00100000:2.0:1787829020.519361:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d26940 x1874670646787968/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829045 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829020.519364:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787829020.519372:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:2.0:1787829020.519375:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:0.0:1787829020.519379:0:40798:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3821 00000100:00000001:2.0:1787829020.519380:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787829020.519386:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829020.519388:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00080000:0.0:1787829020.519389:0:40798:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787829020.519390:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829020.519414:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829020.519414:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787829020.519420:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787829020.519421:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:2.0:1787829020.519424:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000020:00000001:0.0:1787829020.519431:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000100:00080000:2.0:1787829020.519434:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000020:00000001:0.0:1787829020.519438:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:2.0:1787829020.519439:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000020:00000001:0.0:1787829020.519444:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00100000:2.0:1787829020.519448:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d26940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646787968:192.168.203.143@tcp:250:kworker.0 00000020:00000001:0.0:1787829020.519453:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:2.0:1787829020.519456:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829020.519460:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829020.519462:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787829020.519467:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:2.0:1787829020.519467:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787829020.519472:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1787829020.519478:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00080000:0.0:1787829020.519482:0:40798:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000020:00000001:2.0:1787829020.519482:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:2.0:1787829020.519486:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:2.0:1787829020.519492:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 00000100:00000001:0.0:1787829020.519493:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829020.519495:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 02000000:00000001:0.0:1787829020.519501:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:2.0:1787829020.519503:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787829020.519506:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:0.0:1787829020.519509:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787829020.519510:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:0.0:1787829020.519519:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829020.519522:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787829020.519524:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1787829020.519544:0:40798:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1787829020.519548:0:40798:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829020.519550:0:40798:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829020.519550:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829020.519555:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:0.0:1787829020.519556:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:2.0:1787829020.519557:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 02000000:00000001:0.0:1787829020.519562:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829020.519562:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1787829020.519565:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787829020.519568:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:2.0:1787829020.519568:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829020.519575:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787829020.519575:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787829020.519579:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787829020.519581:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829020.519582:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787829020.519590:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d25680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646788096:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787829020.519592:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829020.519593:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787829020.519596:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000100:00100000:0.0:1787829020.519604:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d27c00 x1874670646788352/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787829020.519614:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787829020.519633:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829020.519637:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829020.519639:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829020.519643:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1787829020.519648:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829020.519656:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787829020.519658:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601342080 : -126234108209536 : ffff8d30d4d25680) 00000100:00000001:2.0:1787829020.519663:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1787829020.519703:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:2.0:1787829020.519709:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1787829020.519713:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787829020.519715:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829020.519718:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1787829020.519722:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829020.519728:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298493463 00000100:00000001:2.0:1787829020.519732:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298494720 00000100:00000001:2.0:1787829020.519734:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298493463 00000100:00000001:2.0:1787829020.519737:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298494720 00000100:00000001:2.0:1787829020.519739:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787829020.519742:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829020.519744:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1787829020.519750:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829020/real 1787829020] req@ffff8d30d4d25680 x1874670646788096/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829045 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1787829020.519760:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787829020.519761:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1787829020.519764:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1787829020.519765:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787829020.519766:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829020.519768:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787829020.519784:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787829020.519787:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d25680 x1874670646788096/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829045 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787829020.519792:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829020.519794:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787829020.519795:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829020.519797:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787829020.519797:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829020.519799:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787829020.519800:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787829020.519801:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829020.519803:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:2.0:1787829020.519805:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1787829020.519807:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d25680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646788096:192.168.203.143@tcp:38:kworker.0 00000100:00000001:2.0:1787829020.519810:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1787829020.519811:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787829020.519812:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829020.519814:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787829020.519815:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787829020.519816:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787829020.519818:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d252c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646788224:192.168.203.143@tcp:38:kworker.0 00000100:00000001:2.0:1787829020.519820:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787829020.519822:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1787829020.519823:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829020.519825:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1787829020.519827:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829020.519831:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787829020.519832:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601341120 : -126234108210496 : ffff8d30d4d252c0) 00000100:00000001:2.0:1787829020.519851:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1787829020.519867:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:2.0:1787829020.519873:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1787829020.519875:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787829020.519877:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829020.519879:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1787829020.519881:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829020.519885:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298493463 00000100:00000001:2.0:1787829020.519888:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298494720 00000100:00000001:2.0:1787829020.519891:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298493463 00000100:00000001:2.0:1787829020.519893:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298494720 00000100:00000001:2.0:1787829020.519895:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787829020.519897:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829020.519917:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1787829020.519922:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829020/real 1787829020] req@ffff8d30d4d252c0 x1874670646788224/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829045 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1787829020.519940:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787829020.519942:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1787829020.519945:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1787829020.519947:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787829020.519948:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829020.519950:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787829020.519952:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787829020.519984:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d252c0 x1874670646788224/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829045 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787829020.519994:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829020.519997:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787829020.520001:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829020.520003:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787829020.520021:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829020.520023:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787829020.520024:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787829020.520025:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829020.520028:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:2.0:1787829020.520029:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1787829020.520032:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d252c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646788224:192.168.203.143@tcp:38:kworker.0 00000100:00000001:2.0:1787829020.520034:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829020.520036:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787829020.520037:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787829020.520038:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1787829020.520040:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787829020.520041:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:2.0:1787829020.520042:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:2.0:1787829020.520044:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787829020.520045:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1787829020.520047:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787829020.520048:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829020.520049:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787829020.520050:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787829020.520051:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1787829020.520052:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787829020.520053:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:2.0:1787829020.520054:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:2.0:1787829020.520055:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787829020.520056:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1787829020.520057:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787829020.520058:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829020.520059:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829020.520061:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1787829020.520062:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829020.520064:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787829020.520065:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1787829020.520066:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1787829020.520066:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787829020.520067:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829020.520069:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787829020.520070:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787829020.520072:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787829020.520074:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d27c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646788352:192.168.203.143@tcp:8:kworker.0 00000100:00000001:2.0:1787829020.520076:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787829020.520078:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1787829020.520079:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829020.520080:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1787829020.520082:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829020.520086:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787829020.520087:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601351680 : -126234108199936 : ffff8d30d4d27c00) 00000100:00000001:2.0:1787829020.520090:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1787829020.520099:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:2.0:1787829020.520102:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1787829020.520104:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787829020.520105:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829020.520106:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1787829020.520108:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829020.520110:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298493476 00000100:00000001:2.0:1787829020.520112:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298494720 00000100:00000001:2.0:1787829020.520113:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298493476 00000100:00000001:2.0:1787829020.520115:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298494720 00000100:00000001:2.0:1787829020.520117:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787829020.520118:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829020.520119:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1787829020.520122:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829020/real 1787829020] req@ffff8d30d4d27c00 x1874670646788352/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829045 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1787829020.520127:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787829020.520128:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1787829020.520129:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1787829020.520130:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787829020.520132:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829020.520133:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787829020.520134:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787829020.520136:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d27c00 x1874670646788352/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829045 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787829020.520140:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829020.520142:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787829020.520143:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829020.520144:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787829020.520145:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829020.520146:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787829020.520148:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787829020.520149:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829020.520151:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:2.0:1787829020.520152:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1787829020.520154:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d27c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646788352:192.168.203.143@tcp:8:kworker.0 00000100:00000001:2.0:1787829020.520157:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829020.520158:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787829020.520160:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787829020.520161:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1787829020.520162:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787829020.520163:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:2.0:1787829020.520164:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:2.0:1787829020.520165:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787829020.520166:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1787829020.520168:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787829020.520169:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829020.520171:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829020.520172:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1787829020.520174:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829020.520175:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787829020.520176:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829021.414720:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829021.414727:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829021.414817:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829021.414821:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829021.414824:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829022.438700:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829022.438709:0:815:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1787829022.438712:0:815:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1787829022.438715:0:815:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829022.438720:0:815:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1787829022.438722:0:815:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829022.438724:0:815:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829022.438742:0:815:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829022.438847:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829022.438853:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829022.438857:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829023.462715:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829023.462722:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829023.462781:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829023.462786:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829023.462807:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829024.486738:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829024.486745:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829024.486813:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829024.486821:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829024.486826:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829025.510727:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829025.510750:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829025.510825:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829025.510830:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829025.510833:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787829025.638697:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829025.638715:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1787829025.638717:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787829025.638721:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787829025.638726:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787829025.638734:0:40798:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 3826 00000100:00080000:0.0:1787829025.638746:0:40798:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787829025.638749:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829025.638751:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787829025.638755:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787829025.638758:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787829025.638761:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:0.0:1787829025.638762:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:0.0:1787829025.638764:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829025.638765:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:0.0:1787829025.638769:0:40798:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787829025.638771:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829025.638774:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787829025.638777:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787829025.638779:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829025.638781:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1787829025.638783:0:40798:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829025.638796:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787829025.638797:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829025.638800:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787829025.638803:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787829025.638807:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829025.638819:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787829025.638830:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d270c0 x1874670646788480/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787829025.638844:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787829025.638849:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829025.638866:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787829025.638870:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829025.638873:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1787829025.638874:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787829025.638876:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787829025.638879:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787829025.638881:0:40798:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3826 00000100:00080000:0.0:1787829025.638886:0:40798:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787829025.638888:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829025.638890:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787829025.638893:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787829025.638895:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787829025.638897:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000020:00000001:0.0:1787829025.638900:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:0.0:1787829025.638903:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829025.638904:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:0.0:1787829025.638908:0:40798:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787829025.638911:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829025.638914:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:2.0:1787829025.638930:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787829025.638937:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1787829025.638942:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1787829025.638945:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787829025.638947:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829025.638952:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787829025.638956:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787829025.638959:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787829025.638965:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d270c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646788480:192.168.203.143@tcp:250:kworker.0 00000100:00000001:2.0:1787829025.638973:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787829025.638978:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1787829025.638981:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829025.638984:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1787829025.638990:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829025.639009:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787829025.639011:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601348800 : -126234108202816 : ffff8d30d4d270c0) 00000100:00000001:2.0:1787829025.639015:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:2.0:1787829025.639072:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829025.639078:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4298499840 00000100:00000001:2.0:1787829025.639090:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 02000000:00000001:0.0:1787829025.639099:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787829025.639100:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4298499840 02000000:00000001:0.0:1787829025.639107:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829025.639110:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1787829025.639111:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:2.0:1787829025.639116:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787829025.639120:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829025.639124:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000008:00100000:0.0:1787829025.639127:0:40798:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00000001:2.0:1787829025.639127:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000008:00000001:0.0:1787829025.639135:0:40798:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829025.639138:0:40798:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829025.639144:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:2.0:1787829025.639145:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 02000000:00000001:0.0:1787829025.639147:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829025.639150:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:0.0:1787829025.639151:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:2.0:1787829025.639155:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:0.0:1787829025.639156:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:2.0:1787829025.639158:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 02000000:00000001:0.0:1787829025.639161:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829025.639162:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829025.639168:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829025.639173:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787829025.639181:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d24000 x1874670646788608/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787829025.639188:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787829025.639190:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829025.639203:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787829025.639205:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829025.639209:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1787829025.639211:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787829025.639213:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787829025.639215:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787829025.639217:0:40798:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3826 00000100:00080000:0.0:1787829025.639220:0:40798:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787829025.639222:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829025.639223:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787829025.639225:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787829025.639227:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787829025.639229:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000020:00000001:0.0:1787829025.639231:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:0.0:1787829025.639233:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829025.639234:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:0.0:1787829025.639237:0:40798:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787829025.639239:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787829025.639240:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829025.639243:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:2.0:1787829025.639243:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1787829025.639247:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 02000000:00000001:0.0:1787829025.639249:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787829025.639251:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 02000000:00000001:0.0:1787829025.639253:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829025.639255:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829025.639256:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 02000000:00000001:2.0:1787829025.639260:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787829025.639263:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000008:00100000:0.0:1787829025.639265:0:40798:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 02000000:00000001:2.0:1787829025.639267:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:0.0:1787829025.639269:0:40798:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829025.639272:0:40798:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787829025.639276:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d24000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646788608:192.168.203.143@tcp:38:kworker.0 02000000:00000001:0.0:1787829025.639278:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787829025.639281:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829025.639284:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:0.0:1787829025.639285:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787829025.639289:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787829025.639290:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787829025.639291:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829025.639294:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829025.639297:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829025.639299:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00100000:0.0:1787829025.639304:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d24f00 x1874670646788736/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:2.0:1787829025.639305:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829025.639314:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787829025.639316:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829025.639318:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787829025.639320:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829025.639322:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601336320 : -126234108215296 : ffff8d30d4d24000) 00000100:00080000:0.0:1787829025.639323:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829025.639329:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1787829025.639330:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:2.0:1787829025.639331:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00080000:0.0:1787829025.639332:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787829025.639337:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787829025.639339:0:40798:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3826 00000100:00080000:0.0:1787829025.639342:0:40798:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787829025.639348:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829025.639349:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829025.639350:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787829025.639355:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787829025.639356:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787829025.639358:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:2.0:1787829025.639358:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298498582 00000020:00000001:0.0:1787829025.639363:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:2.0:1787829025.639365:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298499840 00000100:00000001:0.0:1787829025.639366:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829025.639369:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:2.0:1787829025.639373:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298498582 00000100:00080000:0.0:1787829025.639374:0:40798:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787829025.639380:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829025.639380:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298499840 02000000:00000001:0.0:1787829025.639383:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787829025.639387:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787829025.639387:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787829025.639394:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829025.639394:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829025.639399:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:2.0:1787829025.639401:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000008:00100000:0.0:1787829025.639402:0:40798:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00000001:2.0:1787829025.639408:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000008:00000001:0.0:1787829025.639411:0:40798:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829025.639415:0:40798:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829025.639415:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1787829025.639419:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 02000000:00000001:0.0:1787829025.639424:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:2.0:1787829025.639424:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:0.0:1787829025.639428:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829025.639429:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787829025.639431:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:2.0:1787829025.639433:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 02000000:00000001:0.0:1787829025.639434:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:2.0:1787829025.639438:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 02000000:00000001:0.0:1787829025.639441:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829025.639442:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829025.639447:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787829025.639452:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d25a40 x1874670646788864/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:2.0:1787829025.639457:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787829025.639461:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787829025.639466:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829025.639473:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00100000:2.0:1787829025.639475:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d24f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646788736:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787829025.639479:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829025.639482:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:0.0:1787829025.639487:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829025.639487:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1787829025.639490:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829025.639494:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1787829025.639498:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829025.639504:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787829025.639506:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601340160 : -126234108211456 : ffff8d30d4d24f00) 00000100:00000001:2.0:1787829025.639511:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:2.0:1787829025.639524:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829025.639528:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298498582 00000100:00000001:2.0:1787829025.639531:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298499840 00000100:00000001:2.0:1787829025.639533:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298498582 00000100:00000001:2.0:1787829025.639535:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298499840 00000100:00000001:2.0:1787829025.639538:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787829025.639540:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829025.639542:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829025.639545:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829025.639547:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1787829025.639549:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:2.0:1787829025.639551:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787829025.639553:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1787829025.639554:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1787829025.639555:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787829025.639557:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829025.639559:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787829025.639560:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787829025.639562:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787829025.639566:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d25a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646788864:192.168.203.143@tcp:8:kworker.0 00000100:00000001:2.0:1787829025.639570:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787829025.639571:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1787829025.639573:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829025.639575:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1787829025.639578:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829025.639590:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787829025.639592:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601343040 : -126234108208576 : ffff8d30d4d25a40) 00000100:00000001:2.0:1787829025.639595:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:2.0:1787829025.639638:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829025.639642:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298498596 00000100:00000001:2.0:1787829025.639644:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298499840 00000100:00000001:2.0:1787829025.639647:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298498596 00000100:00000001:2.0:1787829025.639650:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298499840 00000100:00000001:2.0:1787829025.639653:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787829025.639655:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829025.639658:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829025.639660:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829025.639663:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1787829025.639664:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:2.0:1787829025.639666:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787829025.639668:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1787829025.639669:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829025.639671:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829026.534743:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829026.534750:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829026.534817:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829026.534821:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829026.534825:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829027.558671:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829027.558676:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829027.558750:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829027.558754:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829027.558757:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829028.582711:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829028.582717:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829028.582766:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829028.582770:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829028.582772:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:1.0:1787829028.710935:0:2378:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.143:988 00000400:00000100:1.0:1787829028.710996:0:2378:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.143@tcp at host 192.168.203.143 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1787829028.711019:0:2378:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.143@tcp 00000100:00000001:1.0:1787829028.711038:0:2378:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787829028.711066:0:2378:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829028.711069:0:2378:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829028.711072:0:2378:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:1.0:1787829028.711085:0:2378:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.143@tcp 00000100:00000001:1.0:1787829028.711109:0:2378:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787829028.711113:0:2378:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829028.711115:0:2378:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829028.711119:0:2378:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787829028.711121:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000800:00000100:1.0:1787829028.711131:0:2378:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.143@tcp 00000100:00000001:0.0:1787829028.711132:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787829028.711140:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00000001:1.0:1787829028.711143:0:2378:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787829028.711151:0:2378:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829028.711153:0:2378:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829028.711157:0:2378:0:(events.c:68:request_out_callback()) Process leaving 00000100:00100000:0.0:1787829028.711163:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829025/real 1787829028] req@ffff8d30d4d24000 x1874670646788608/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829050 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000100:1.0:1787829028.711171:0:2378:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.143@tcp 00000100:00000001:1.0:1787829028.711186:0:2378:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787829028.711192:0:2378:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000400:00000001:0.0:1787829028.711194:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000001:1.0:1787829028.711195:0:2378:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:0.0:1787829028.711201:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787829028.711202:0:2378:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787829028.711224:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787829028.711230:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787829028.711233:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829028.711237:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787829028.711239:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787829028.711245:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d24000 x1874670646788608/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829050 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787829028.711254:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829028.711257:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787829028.711261:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829028.711262:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787829028.711264:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829028.711266:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787829028.711269:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787829028.711274:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829028.711279:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787829028.711282:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787829028.711286:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d24000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646788608:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787829028.711291:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787829028.711298:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829025/real 1787829028] req@ffff8d30d4d270c0 x1874670646788480/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829050 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787829028.711306:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787829028.711308:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787829028.711311:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787829028.711313:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787829028.711314:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829028.711315:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787829028.711317:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787829028.711321:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d270c0 x1874670646788480/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829050 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787829028.711327:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829028.711329:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787829028.711332:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829028.711333:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787829028.711335:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829028.711336:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787829028.711338:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787829028.711340:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829028.711344:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787829028.711347:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787829028.711351:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d270c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646788480:192.168.203.143@tcp:250:kworker.0 00000100:00000001:0.0:1787829028.711355:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829028.711358:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787829028.711361:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787829028.711364:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787829028.711372:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787829028.711375:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787829028.711378:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787829028.711382:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787829028.711384:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787829028.711388:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787829028.711389:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829028.711392:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787829028.711393:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787829028.711394:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787829028.711397:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787829028.711398:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787829028.711399:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787829028.711401:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787829028.711402:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787829028.711405:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787829028.711406:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829028.711409:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829028.711411:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787829028.711413:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787829028.711414:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787829028.711419:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829025/real 1787829028] req@ffff8d30d4d25a40 x1874670646788864/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829050 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787829028.711429:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787829028.711431:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787829028.711434:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787829028.711436:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787829028.711437:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829028.711439:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787829028.711441:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787829028.711445:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d25a40 x1874670646788864/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829050 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787829028.711452:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829028.711454:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787829028.711456:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829028.711458:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787829028.711460:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829028.711462:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787829028.711464:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787829028.711466:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829028.711470:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787829028.711473:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787829028.711477:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d25a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646788864:192.168.203.143@tcp:8:kworker.0 00000100:00000001:0.0:1787829028.711482:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787829028.711486:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829025/real 1787829028] req@ffff8d30d4d24f00 x1874670646788736/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829050 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787829028.711494:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787829028.711495:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787829028.711499:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787829028.711500:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787829028.711502:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829028.711504:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787829028.711505:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787829028.711509:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d24f00 x1874670646788736/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829050 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787829028.711516:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829028.711518:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787829028.711520:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829028.711522:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787829028.711523:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829028.711525:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787829028.711527:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787829028.711529:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829028.711533:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787829028.711535:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787829028.711538:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d24f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646788736:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787829028.711542:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829028.711544:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787829028.711545:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787829028.711547:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787829028.711550:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787829028.711551:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787829028.711553:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787829028.711555:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787829028.711556:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787829028.711559:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787829028.711560:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829028.711562:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787829028.711563:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787829028.711565:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787829028.711567:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787829028.711568:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787829028.711569:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787829028.711572:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787829028.711573:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787829028.711574:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787829028.711576:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829028.711577:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829028.711585:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787829028.711587:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829028.711590:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787829028.711591:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829029.606687:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829029.606694:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829029.606748:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829029.606751:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829029.606754:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829030.630681:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829030.630687:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829030.630736:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829030.630740:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829030.630745:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787829030.758713:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829030.758735:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1787829030.758738:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787829030.758743:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787829030.758748:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787829030.758756:0:40798:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 3831 00000100:00080000:0.0:1787829030.758768:0:40798:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787829030.758772:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829030.758774:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787829030.758779:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787829030.758783:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787829030.758787:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:0.0:1787829030.758790:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:0.0:1787829030.758793:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829030.758794:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:0.0:1787829030.758810:0:40798:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787829030.758814:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829030.758817:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787829030.758821:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787829030.758824:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829030.758827:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1787829030.758830:0:40798:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829030.758839:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787829030.758841:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829030.758844:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787829030.758847:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787829030.758851:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829030.758863:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787829030.758873:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d24f00 x1874670646788992/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787829030.758910:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787829030.758915:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829030.758933:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787829030.758937:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829030.758941:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1787829030.758943:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787829030.758944:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787829030.758951:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787829030.758954:0:40798:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3831 00000100:00080000:0.0:1787829030.758958:0:40798:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787829030.758961:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829030.758963:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787829030.758966:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787829030.758968:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787829030.758970:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000020:00000001:0.0:1787829030.758973:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:0.0:1787829030.758976:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829030.758978:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:2.0:1787829030.758980:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00080000:0.0:1787829030.758983:0:40798:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787829030.758986:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787829030.758989:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829030.758991:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 02000000:00000001:0.0:1787829030.758993:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:2.0:1787829030.758995:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787829030.758999:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829030.759004:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787829030.759005:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787829030.759008:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787829030.759011:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829030.759013:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829030.759016:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1787829030.759023:0:40798:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829030.759024:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d24f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646788992:192.168.203.143@tcp:250:kworker.0 00000008:00000001:0.0:1787829030.759030:0:40798:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829030.759033:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:0.0:1787829030.759035:0:40798:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829030.759039:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787829030.759043:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787829030.759044:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829030.759047:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829030.759049:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:0.0:1787829030.759051:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787829030.759055:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787829030.759056:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829030.759058:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829030.759068:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787829030.759075:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d25a40 x1874670646789120/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787829030.759076:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787829030.759080:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601340160 : -126234108211456 : ffff8d30d4d24f00) 00000100:00000001:2.0:1787829030.759088:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:0.0:1787829030.759093:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787829030.759097:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829030.759101:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787829030.759104:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829030.759108:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1787829030.759109:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787829030.759112:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787829030.759115:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787829030.759119:0:40798:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3831 00000100:00080000:0.0:1787829030.759124:0:40798:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787829030.759126:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829030.759128:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000800:00000100:2.0:1787829030.759132:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000020:00000001:0.0:1787829030.759134:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787829030.759137:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787829030.759140:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000020:00000001:0.0:1787829030.759144:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:0.0:1787829030.759146:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829030.759149:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:2.0:1787829030.759149:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1787829030.759156:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00080000:0.0:1787829030.759158:0:40798:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787829030.759161:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829030.759163:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829030.759166:0:2390:0:(events.c:68:request_out_callback()) Process leaving 02000000:00000001:0.0:1787829030.759168:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787829030.759174:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787829030.759175:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829030.759178:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829030.759181:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4298504960 00000100:00000001:0.0:1787829030.759183:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1787829030.759188:0:40798:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1787829030.759191:0:40798:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829030.759193:0:40798:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829030.759193:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 00000100:00000001:2.0:1787829030.759201:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4298504960 02000000:00000001:0.0:1787829030.759203:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787829030.759206:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829030.759209:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1787829030.759210:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787829030.759214:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:2.0:1787829030.759216:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787829030.759217:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829030.759221:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829030.759225:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00000001:0.0:1787829030.759227:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787829030.759238:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d270c0 x1874670646789248/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787829030.759239:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829030/real 1787829030] req@ffff8d30d4d24f00 x1874670646788992/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829055 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787829030.759262:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000400:00000001:2.0:1787829030.759265:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000001:0.0:1787829030.759267:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:2.0:1787829030.759269:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787829030.759273:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787829030.759278:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829030.759282:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1787829030.759283:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787829030.759284:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787829030.759287:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:2.0:1787829030.759288:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00080000:0.0:1787829030.759293:0:40798:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3831 00000100:00000001:2.0:1787829030.759294:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00080000:0.0:1787829030.759300:0:40798:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787829030.759300:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829030.759304:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787829030.759306:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829030.759309:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829030.759310:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787829030.759315:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787829030.759319:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00100000:2.0:1787829030.759320:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d24f00 x1874670646788992/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829055 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:0.0:1787829030.759323:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000020:00000001:0.0:1787829030.759329:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:0.0:1787829030.759335:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829030.759338:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829030.759339:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:2.0:1787829030.759343:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787829030.759348:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:0.0:1787829030.759349:0:40798:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787829030.759352:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787829030.759357:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829030.759358:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829030.759364:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 02000000:00000001:0.0:1787829030.759365:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:2.0:1787829030.759368:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 02000000:00000001:0.0:1787829030.759370:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00080000:2.0:1787829030.759373:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 02000000:00000001:0.0:1787829030.759376:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829030.759381:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00080000:2.0:1787829030.759382:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:2.0:1787829030.759388:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000008:00100000:0.0:1787829030.759389:0:40798:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:2.0:1787829030.759394:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d24f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646788992:192.168.203.143@tcp:250:kworker.0 00000008:00000001:0.0:1787829030.759395:0:40798:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829030.759400:0:40798:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829030.759406:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:0.0:1787829030.759407:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787829030.759411:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829030.759413:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787829030.759416:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:2.0:1787829030.759417:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787829030.759422:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787829030.759423:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787829030.759427:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829030.759429:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787829030.759434:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000100:00000001:0.0:1787829030.759435:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1787829030.759437:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:2.0:1787829030.759444:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 00000100:00100000:0.0:1787829030.759446:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d24000 x1874670646789376/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:2.0:1787829030.759448:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1787829030.759454:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787829030.759457:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829030.759459:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787829030.759462:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829030.759465:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829030.759470:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829030.759477:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1787829030.759481:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829030.759485:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787829030.759486:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1787829030.759488:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1787829030.759490:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787829030.759493:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829030.759495:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787829030.759498:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787829030.759500:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787829030.759504:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d25a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646789120:192.168.203.143@tcp:38:kworker.0 00000100:00000001:2.0:1787829030.759508:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787829030.759510:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1787829030.759511:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829030.759513:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1787829030.759516:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829030.759522:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787829030.759523:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601343040 : -126234108208576 : ffff8d30d4d25a40) 00000100:00000001:2.0:1787829030.759528:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1787829030.759541:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:2.0:1787829030.759547:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1787829030.759549:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787829030.759551:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829030.759553:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1787829030.759556:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829030.759560:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298503702 00000100:00000001:2.0:1787829030.759563:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298504960 00000100:00000001:2.0:1787829030.759566:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298503702 00000100:00000001:2.0:1787829030.759568:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298504960 00000100:00000001:2.0:1787829030.759570:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787829030.759572:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829030.759574:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1787829030.759579:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829030/real 1787829030] req@ffff8d30d4d25a40 x1874670646789120/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829055 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1787829030.759587:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787829030.759589:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1787829030.759592:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1787829030.759593:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787829030.759595:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829030.759597:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787829030.759598:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787829030.759627:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d25a40 x1874670646789120/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829055 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787829030.759635:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829030.759637:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787829030.759640:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829030.759642:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787829030.759643:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829030.759645:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787829030.759647:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787829030.759650:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829030.759653:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:2.0:1787829030.759656:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1787829030.759659:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d25a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646789120:192.168.203.143@tcp:38:kworker.0 00000100:00000001:2.0:1787829030.759664:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1787829030.759665:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787829030.759666:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829030.759669:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787829030.759671:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787829030.759673:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787829030.759675:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d270c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646789248:192.168.203.143@tcp:38:kworker.0 00000100:00000001:2.0:1787829030.759679:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787829030.759680:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1787829030.759682:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829030.759683:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1787829030.759687:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829030.759692:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787829030.759694:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601348800 : -126234108202816 : ffff8d30d4d270c0) 00000100:00000001:2.0:1787829030.759697:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1787829030.759709:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:2.0:1787829030.759715:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1787829030.759717:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787829030.759718:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829030.759720:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1787829030.759723:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829030.759726:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298503703 00000100:00000001:2.0:1787829030.759729:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298504960 00000100:00000001:2.0:1787829030.759731:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298503703 00000100:00000001:2.0:1787829030.759733:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298504960 00000100:00000001:2.0:1787829030.759735:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787829030.759737:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829030.759739:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1787829030.759744:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829030/real 1787829030] req@ffff8d30d4d270c0 x1874670646789248/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829055 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1787829030.759758:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787829030.759759:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1787829030.759763:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1787829030.759765:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787829030.759766:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829030.759768:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787829030.759769:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787829030.759773:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d270c0 x1874670646789248/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829055 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787829030.759781:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829030.759784:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787829030.759787:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829030.759788:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787829030.759790:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829030.759792:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787829030.759794:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787829030.759797:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829030.759801:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:2.0:1787829030.759804:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1787829030.759807:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d270c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646789248:192.168.203.143@tcp:38:kworker.0 00000100:00000001:2.0:1787829030.759812:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1787829030.759813:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787829030.759815:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829030.759817:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787829030.759819:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787829030.759821:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787829030.759824:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d24000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646789376:192.168.203.143@tcp:8:kworker.0 00000100:00000001:2.0:1787829030.759828:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787829030.759830:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1787829030.759832:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829030.759834:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1787829030.759837:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829030.759843:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787829030.759844:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601336320 : -126234108215296 : ffff8d30d4d24000) 00000100:00000001:2.0:1787829030.759848:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1787829030.759860:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:2.0:1787829030.759865:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1787829030.759868:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787829030.759869:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829030.759871:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1787829030.759873:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829030.759877:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298503716 00000100:00000001:2.0:1787829030.759879:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298504960 00000100:00000001:2.0:1787829030.759881:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298503716 00000100:00000001:2.0:1787829030.759902:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298504960 00000100:00000001:2.0:1787829030.759905:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787829030.759907:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829030.759909:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1787829030.759914:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829030/real 1787829030] req@ffff8d30d4d24000 x1874670646789376/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829055 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1787829030.759922:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787829030.759924:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1787829030.759927:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1787829030.759929:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787829030.759930:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829030.759932:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787829030.759933:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787829030.759937:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d24000 x1874670646789376/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829055 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787829030.759943:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829030.759945:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787829030.759948:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829030.759950:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787829030.759951:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829030.759953:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787829030.759955:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787829030.759957:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829030.759961:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:2.0:1787829030.759962:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1787829030.759965:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d24000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646789376:192.168.203.143@tcp:8:kworker.0 00000100:00000001:2.0:1787829030.759967:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829030.759969:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787829030.759970:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787829030.759971:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1787829030.759973:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787829030.759974:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:2.0:1787829030.759975:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:2.0:1787829030.759977:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787829030.759978:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1787829030.759980:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787829030.759980:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829030.759982:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787829030.759982:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787829030.759983:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1787829030.759985:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787829030.759986:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:2.0:1787829030.759986:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:2.0:1787829030.759988:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787829030.759989:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1787829030.759992:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787829030.759993:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829030.759994:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787829030.759995:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787829030.759997:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1787829030.759998:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787829030.760000:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:2.0:1787829030.760001:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:2.0:1787829030.760002:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787829030.760004:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1787829030.760005:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787829030.760006:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829030.760008:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829030.760012:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1787829030.760014:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829030.760017:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787829030.760018:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829031.654681:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829031.654687:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829031.654733:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829031.654736:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829031.654739:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829032.678710:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829032.678717:0:815:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1787829032.678719:0:815:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1787829032.678721:0:815:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829032.678725:0:815:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1787829032.678727:0:815:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829032.678729:0:815:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829032.678730:0:815:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829032.678756:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829032.678759:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829032.678763:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829033.702715:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829033.702722:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829033.702771:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829033.702774:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829033.702776:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829034.726705:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829034.726716:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829034.726793:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829034.726801:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829034.726806:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829035.750694:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829035.750700:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829035.750774:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829035.750777:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829035.750780:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787829035.878713:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829035.878724:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1787829035.878727:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787829035.878730:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787829035.878735:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787829035.878743:0:40798:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 3836 00000100:00080000:0.0:1787829035.878766:0:40798:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787829035.878770:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829035.878772:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787829035.878776:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787829035.878779:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787829035.878782:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:0.0:1787829035.878785:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:0.0:1787829035.878787:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829035.878789:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:0.0:1787829035.878794:0:40798:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787829035.878797:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829035.878800:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787829035.878804:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787829035.878807:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829035.878809:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1787829035.878812:0:40798:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829035.878824:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787829035.878826:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829035.878829:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787829035.878832:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787829035.878836:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829035.878863:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787829035.878873:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d24000 x1874670646789504/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787829035.878884:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787829035.878888:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829035.878904:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787829035.878908:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829035.878911:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1787829035.878913:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787829035.878915:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787829035.878919:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787829035.878922:0:40798:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3836 00000100:00080000:0.0:1787829035.878926:0:40798:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787829035.878929:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829035.878931:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787829035.878934:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787829035.878936:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787829035.878939:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:2.0:1787829035.878940:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:0.0:1787829035.878942:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:2.0:1787829035.878945:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787829035.878947:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829035.878949:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1787829035.878950:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:2.0:1787829035.878952:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787829035.878956:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787829035.878958:0:40798:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 02000000:00000001:2.0:1787829035.878960:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:0.0:1787829035.878962:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829035.878964:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787829035.878967:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787829035.878968:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829035.878974:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00100000:2.0:1787829035.878974:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d24000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646789504:192.168.203.143@tcp:250:kworker.0 02000000:00000001:0.0:1787829035.878981:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829035.878982:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:0.0:1787829035.878986:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 02000000:00000001:2.0:1787829035.878987:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000008:00100000:0.0:1787829035.878991:0:40798:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 02000000:00000001:2.0:1787829035.878991:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829035.878995:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000008:00000001:0.0:1787829035.878998:0:40798:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829035.879001:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829035.879002:0:40798:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829035.879009:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787829035.879011:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829035.879013:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787829035.879015:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:2.0:1787829035.879016:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 02000000:00000001:0.0:1787829035.879018:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829035.879019:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601336320 : -126234108215296 : ffff8d30d4d24000) 00000100:00000001:2.0:1787829035.879025:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:0.0:1787829035.879029:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787829035.879037:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d270c0 x1874670646789632/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787829035.879053:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787829035.879055:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829035.879059:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787829035.879063:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829035.879067:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1787829035.879069:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787829035.879070:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787829035.879073:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787829035.879079:0:40798:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3836 00000800:00000100:2.0:1787829035.879079:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00080000:0.0:1787829035.879086:0:40798:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787829035.879089:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829035.879091:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:2.0:1787829035.879092:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000020:00000001:0.0:1787829035.879094:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:2.0:1787829035.879097:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000020:00000001:0.0:1787829035.879098:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:2.0:1787829035.879100:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787829035.879102:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:2.0:1787829035.879103:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000020:00000001:0.0:1787829035.879108:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:2.0:1787829035.879109:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829035.879114:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4298510080 00000100:00000001:0.0:1787829035.879121:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829035.879122:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1787829035.879126:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:2.0:1787829035.879129:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4298510080 00000100:00080000:0.0:1787829035.879133:0:40798:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787829035.879135:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1787829035.879140:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829035.879141:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787829035.879146:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:2.0:1787829035.879156:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829035.879159:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787829035.879159:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 02000000:00000001:0.0:1787829035.879165:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787829035.879166:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829035/real 1787829035] req@ffff8d30d4d24000 x1874670646789504/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829060 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787829035.879170:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1787829035.879174:0:40798:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1787829035.879179:0:40798:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:2.0:1787829035.879180:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787829035.879183:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787829035.879184:0:40798:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829035.879186:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1787829035.879189:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787829035.879190:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:0.0:1787829035.879191:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:2.0:1787829035.879192:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 02000000:00000001:0.0:1787829035.879194:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829035.879195:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829035.879200:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00100000:2.0:1787829035.879203:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d24000 x1874670646789504/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829060 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:0.0:1787829035.879204:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787829035.879207:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829035.879213:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829035.879215:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829035.879216:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787829035.879220:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829035.879222:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787829035.879224:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787829035.879225:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d25a40 x1874670646789760/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787829035.879226:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787829035.879229:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787829035.879234:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829035.879251:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:2.0:1787829035.879253:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1787829035.879256:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d24000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646789504:192.168.203.143@tcp:250:kworker.0 00000100:00000001:2.0:1787829035.879262:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829035.879266:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787829035.879269:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787829035.879272:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 00000100:00000001:0.0:1787829035.879275:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 02000000:00000001:2.0:1787829035.879275:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787829035.879279:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000100:00000001:0.0:1787829035.879280:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1787829035.879281:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 00000100:00000001:0.0:1787829035.879284:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829035.879287:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 00000100:00080000:0.0:1787829035.879289:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 02000000:00000001:2.0:1787829035.879289:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1787829035.879293:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00080000:0.0:1787829035.879295:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1787829035.879295:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829035.879300:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:2.0:1787829035.879300:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:0.0:1787829035.879305:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787829035.879309:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:2.0:1787829035.879309:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1787829035.879311:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829035.879314:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00080000:0.0:1787829035.879316:0:40798:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3836 00000100:00000001:2.0:1787829035.879317:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1787829035.879320:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1787829035.879323:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787829035.879324:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787829035.879325:0:40798:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 02000000:00000001:2.0:1787829035.879326:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787829035.879329:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787829035.879332:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829035.879340:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00100000:2.0:1787829035.879342:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d270c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646789632:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787829035.879344:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:2.0:1787829035.879345:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787829035.879349:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000020:00000001:0.0:1787829035.879350:0:40798:0:(genops.c:792:class_conn2export()) Process entered 02000000:00000001:2.0:1787829035.879351:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787829035.879354:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 02000000:00000001:2.0:1787829035.879354:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1787829035.879357:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829035.879361:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000020:00000001:0.0:1787829035.879364:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:2.0:1787829035.879364:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601348800 : -126234108202816 : ffff8d30d4d270c0) 00000020:00000001:0.0:1787829035.879371:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:2.0:1787829035.879372:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:0.0:1787829035.879376:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829035.879378:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:0.0:1787829035.879383:0:40798:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000800:00000100:2.0:1787829035.879384:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:0.0:1787829035.879389:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829035.879392:0:2390:0:(events.c:34:request_out_callback()) Process entered 02000000:00000001:0.0:1787829035.879393:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:2.0:1787829035.879398:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 02000000:00000001:0.0:1787829035.879399:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787829035.879412:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829035.879413:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829035.879419:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:2.0:1787829035.879419:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000008:00100000:0.0:1787829035.879424:0:40798:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00000001:2.0:1787829035.879426:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:0.0:1787829035.879427:0:40798:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829035.879431:0:40798:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829035.879433:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298508822 02000000:00000001:0.0:1787829035.879436:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:2.0:1787829035.879439:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298510080 02000000:00000001:0.0:1787829035.879440:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829035.879447:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:2.0:1787829035.879447:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298508822 02000000:00000001:0.0:1787829035.879450:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:2.0:1787829035.879453:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298510080 02000000:00000001:0.0:1787829035.879454:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829035.879459:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787829035.879461:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829035.879463:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829035.879464:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1787829035.879470:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829035/real 1787829035] req@ffff8d30d4d270c0 x1874670646789632/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829060 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829035.879474:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d24f00 x1874670646789888/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000001:2.0:1787829035.879484:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787829035.879487:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1787829035.879490:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787829035.879492:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787829035.879493:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787829035.879497:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829035.879499:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829035.879499:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787829035.879502:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829035.879507:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787829035.879508:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d270c0 x1874670646789632/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829060 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787829035.879517:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829035.879519:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787829035.879521:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829035.879522:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787829035.879523:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829035.879524:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787829035.879525:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787829035.879527:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829035.879529:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:2.0:1787829035.879530:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1787829035.879532:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d270c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646789632:192.168.203.143@tcp:38:kworker.0 00000100:00000001:2.0:1787829035.879535:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1787829035.879536:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787829035.879537:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829035.879539:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787829035.879540:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787829035.879542:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787829035.879544:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d25a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646789760:192.168.203.143@tcp:38:kworker.0 00000100:00000001:2.0:1787829035.879546:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787829035.879548:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1787829035.879549:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829035.879551:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1787829035.879553:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829035.879566:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787829035.879567:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601343040 : -126234108208576 : ffff8d30d4d25a40) 00000100:00000001:2.0:1787829035.879570:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1787829035.879579:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:2.0:1787829035.879582:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1787829035.879584:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787829035.879585:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829035.879586:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1787829035.879588:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829035.879590:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298508822 00000100:00000001:2.0:1787829035.879592:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298510080 00000100:00000001:2.0:1787829035.879593:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298508822 00000100:00000001:2.0:1787829035.879595:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298510080 00000100:00000001:2.0:1787829035.879596:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787829035.879619:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829035.879621:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1787829035.879624:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829035/real 1787829035] req@ffff8d30d4d25a40 x1874670646789760/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829060 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1787829035.879630:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787829035.879631:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1787829035.879633:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1787829035.879634:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787829035.879635:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829035.879636:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787829035.879637:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787829035.879639:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d25a40 x1874670646789760/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829060 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787829035.879644:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829035.879645:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787829035.879646:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829035.879647:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787829035.879648:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829035.879649:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787829035.879650:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787829035.879652:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829035.879654:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:2.0:1787829035.879656:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1787829035.879658:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d25a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646789760:192.168.203.143@tcp:38:kworker.0 00000100:00000001:2.0:1787829035.879660:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829035.879661:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787829035.879662:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787829035.879663:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1787829035.879664:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787829035.879665:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:2.0:1787829035.879666:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:2.0:1787829035.879667:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787829035.879668:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1787829035.879671:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787829035.879672:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829035.879673:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787829035.879674:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787829035.879686:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1787829035.879687:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787829035.879688:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:2.0:1787829035.879689:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:2.0:1787829035.879690:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787829035.879691:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1787829035.879692:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787829035.879693:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829035.879694:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829035.879696:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1787829035.879697:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829035.879698:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787829035.879699:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1787829035.879700:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1787829035.879701:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787829035.879702:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829035.879704:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787829035.879705:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787829035.879706:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787829035.879708:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d24f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646789888:192.168.203.143@tcp:8:kworker.0 00000100:00000001:2.0:1787829035.879711:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787829035.879712:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1787829035.879713:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829035.879714:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1787829035.879716:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829035.879719:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787829035.879721:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601340160 : -126234108211456 : ffff8d30d4d24f00) 00000100:00000001:2.0:1787829035.879723:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1787829035.879730:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:2.0:1787829035.879734:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1787829035.879735:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787829035.879736:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829035.879737:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1787829035.879739:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829035.879751:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298508836 00000100:00000001:2.0:1787829035.879753:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298510080 00000100:00000001:2.0:1787829035.879755:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298508836 00000100:00000001:2.0:1787829035.879756:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298510080 00000100:00000001:2.0:1787829035.879758:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787829035.879759:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829035.879760:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1787829035.879762:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829035/real 1787829035] req@ffff8d30d4d24f00 x1874670646789888/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829060 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1787829035.879767:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787829035.879768:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1787829035.879769:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1787829035.879770:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787829035.879771:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829035.879772:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787829035.879773:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787829035.879776:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d24f00 x1874670646789888/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829060 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787829035.879780:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829035.879781:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787829035.879783:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829035.879783:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787829035.879784:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829035.879785:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787829035.879786:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787829035.879788:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829035.879790:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:2.0:1787829035.879791:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1787829035.879793:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d24f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646789888:192.168.203.143@tcp:8:kworker.0 00000100:00000001:2.0:1787829035.879795:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829035.879796:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787829035.879797:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787829035.879798:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1787829035.879799:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787829035.879800:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:2.0:1787829035.879801:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:2.0:1787829035.879802:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787829035.879803:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1787829035.879804:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787829035.879805:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829035.879806:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829035.879808:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1787829035.879809:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829035.879810:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787829035.879811:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829036.774699:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829036.774704:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829036.774752:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829036.774755:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829036.774758:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829037.798744:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829037.798750:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829037.798862:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829037.798866:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829037.798869:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829038.822705:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829038.822712:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829038.822782:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829038.822786:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829038.822790:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829039.846703:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829039.846710:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829039.846771:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829039.846775:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829039.846861:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829040.870706:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829040.870713:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829040.870769:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829040.870772:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829040.870775:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787829040.998702:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829040.998723:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1787829040.998726:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787829040.998729:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787829040.998732:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787829040.998737:0:40798:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 3841 00000100:00080000:0.0:1787829040.998746:0:40798:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787829040.998748:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829040.998750:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787829040.998753:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787829040.998755:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787829040.998757:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:0.0:1787829040.998759:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:0.0:1787829040.998760:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829040.998761:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:0.0:1787829040.998764:0:40798:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787829040.998766:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829040.998767:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787829040.998770:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787829040.998772:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829040.998774:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1787829040.998776:0:40798:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829040.998784:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787829040.998786:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829040.998789:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787829040.998791:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787829040.998794:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829040.998804:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787829040.998811:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d27c00 x1874670646790016/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787829040.998832:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787829040.998835:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829040.998846:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787829040.998849:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829040.998851:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1787829040.998852:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787829040.998854:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787829040.998856:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787829040.998858:0:40798:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3841 00000100:00080000:0.0:1787829040.998862:0:40798:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787829040.998863:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829040.998865:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787829040.998868:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787829040.998869:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787829040.998871:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000020:00000001:0.0:1787829040.998874:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:0.0:1787829040.998877:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829040.998878:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:0.0:1787829040.998883:0:40798:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787829040.998885:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787829040.998886:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829040.998889:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 02000000:00000001:0.0:1787829040.998891:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:2.0:1787829040.998893:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1787829040.998895:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787829040.998897:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829040.998898:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787829040.998901:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787829040.998902:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829040.998905:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787829040.998907:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 02000000:00000001:2.0:1787829040.998908:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000008:00100000:0.0:1787829040.998913:0:40798:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829040.998914:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d27c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646790016:192.168.203.143@tcp:250:kworker.0 00000008:00000001:0.0:1787829040.998918:0:40798:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829040.998922:0:40798:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829040.998923:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787829040.998929:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787829040.998929:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787829040.998932:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829040.998932:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829040.998937:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787829040.998938:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1787829040.998939:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787829040.998942:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829040.998943:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829040.998947:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787829040.998954:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d252c0 x1874670646790144/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787829040.998959:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787829040.998960:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787829040.998962:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601351680 : -126234108199936 : ffff8d30d4d27c00) 00000100:00000001:0.0:1787829040.998964:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829040.998967:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:0.0:1787829040.998969:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787829040.998972:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829040.998975:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1787829040.998976:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787829040.998977:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787829040.998980:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787829040.998982:0:40798:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3841 00000100:00080000:0.0:1787829040.998985:0:40798:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787829040.998986:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829040.998987:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787829040.998989:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787829040.998990:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787829040.998992:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000020:00000001:0.0:1787829040.998993:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:0.0:1787829040.998995:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829040.998996:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:0.0:1787829040.998998:0:40798:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787829040.999001:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:2.0:1787829040.999002:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 02000000:00000001:0.0:1787829040.999005:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787829040.999008:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787829040.999010:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829040.999012:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1787829040.999014:0:40798:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1787829040.999015:0:40798:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829040.999016:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1787829040.999017:0:40798:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829040.999022:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 02000000:00000001:0.0:1787829040.999023:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787829040.999026:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829040.999028:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829040.999029:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787829040.999031:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:2.0:1787829040.999033:0:2390:0:(events.c:68:request_out_callback()) Process leaving 02000000:00000001:0.0:1787829040.999034:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829040.999038:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829040.999038:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829040.999045:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4298515200 00000100:00100000:0.0:1787829040.999047:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d25680 x1874670646790272/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787829040.999053:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787829040.999055:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1787829040.999056:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829040.999061:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787829040.999063:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:2.0:1787829040.999065:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4298515200 00000100:00080000:0.0:1787829040.999066:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1787829040.999069:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787829040.999071:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787829040.999072:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1787829040.999073:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787829040.999077:0:40798:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3841 00000100:00000001:2.0:1787829040.999078:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00080000:0.0:1787829040.999080:0:40798:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787829040.999081:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829040.999082:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829040.999085:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:2.0:1787829040.999085:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000020:00000001:0.0:1787829040.999088:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787829040.999089:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787829040.999091:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00100000:2.0:1787829040.999093:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829040/real 1787829040] req@ffff8d30d4d27c00 x1874670646790016/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829065 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:0.0:1787829040.999094:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:0.0:1787829040.999098:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829040.999101:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:0.0:1787829040.999108:0:40798:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787829040.999111:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:2.0:1787829040.999112:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 02000000:00000001:0.0:1787829040.999113:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000400:00000001:2.0:1787829040.999116:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 02000000:00000001:0.0:1787829040.999117:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787829040.999120:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829040.999120:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787829040.999124:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:2.0:1787829040.999124:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000008:00100000:0.0:1787829040.999125:0:40798:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00000001:2.0:1787829040.999127:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000008:00000001:0.0:1787829040.999129:0:40798:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829040.999130:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787829040.999132:0:40798:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829040.999133:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:0.0:1787829040.999138:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787829040.999141:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787829040.999141:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d27c00 x1874670646790016/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829065 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787829040.999144:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787829040.999147:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787829040.999149:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829040.999153:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829040.999155:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787829040.999159:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d26940 x1874670646790400/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787829040.999159:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787829040.999165:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829040.999169:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787829040.999171:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829040.999174:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787829040.999175:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787829040.999177:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829040.999179:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787829040.999180:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787829040.999184:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829040.999189:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:2.0:1787829040.999191:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1787829040.999193:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d27c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646790016:192.168.203.143@tcp:250:kworker.0 00000100:00000001:2.0:1787829040.999198:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829040.999199:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787829040.999200:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787829040.999202:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1787829040.999205:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787829040.999207:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:2.0:1787829040.999209:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:2.0:1787829040.999212:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787829040.999214:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1787829040.999216:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787829040.999217:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829040.999219:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829040.999225:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1787829040.999227:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829040.999228:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787829040.999229:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1787829040.999231:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1787829040.999232:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787829040.999233:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829040.999234:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787829040.999236:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787829040.999238:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787829040.999240:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d252c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646790144:192.168.203.143@tcp:38:kworker.0 00000100:00000001:2.0:1787829040.999242:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787829040.999244:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1787829040.999245:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829040.999246:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1787829040.999249:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829040.999254:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787829040.999255:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601341120 : -126234108210496 : ffff8d30d4d252c0) 00000100:00000001:2.0:1787829040.999259:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1787829040.999270:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:2.0:1787829040.999274:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1787829040.999276:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787829040.999277:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829040.999278:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1787829040.999280:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829040.999283:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298513942 00000100:00000001:2.0:1787829040.999285:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298515200 00000100:00000001:2.0:1787829040.999287:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298513942 00000100:00000001:2.0:1787829040.999289:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298515200 00000100:00000001:2.0:1787829040.999290:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787829040.999291:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829040.999293:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1787829040.999296:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829040/real 1787829040] req@ffff8d30d4d252c0 x1874670646790144/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829065 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1787829040.999301:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787829040.999302:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1787829040.999303:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1787829040.999304:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787829040.999305:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829040.999307:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787829040.999308:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787829040.999312:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d252c0 x1874670646790144/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829065 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787829040.999316:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829040.999317:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787829040.999318:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829040.999319:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787829040.999320:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829040.999321:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787829040.999322:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787829040.999325:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829040.999327:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:2.0:1787829040.999328:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1787829040.999331:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d252c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646790144:192.168.203.143@tcp:38:kworker.0 00000100:00000001:2.0:1787829040.999334:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1787829040.999335:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787829040.999336:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829040.999338:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787829040.999339:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787829040.999341:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787829040.999343:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d25680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646790272:192.168.203.143@tcp:38:kworker.0 00000100:00000001:2.0:1787829040.999345:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787829040.999346:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1787829040.999347:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829040.999349:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1787829040.999351:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829040.999355:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787829040.999356:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601342080 : -126234108209536 : ffff8d30d4d25680) 00000100:00000001:2.0:1787829040.999359:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1787829040.999368:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:2.0:1787829040.999372:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1787829040.999374:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787829040.999375:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829040.999377:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1787829040.999378:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829040.999381:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298513942 00000100:00000001:2.0:1787829040.999382:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298515200 00000100:00000001:2.0:1787829040.999384:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298513942 00000100:00000001:2.0:1787829040.999386:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298515200 00000100:00000001:2.0:1787829040.999388:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787829040.999389:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829040.999391:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1787829040.999394:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829040/real 1787829040] req@ffff8d30d4d25680 x1874670646790272/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829065 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1787829040.999399:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787829040.999400:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1787829040.999402:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1787829040.999403:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787829040.999403:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829040.999404:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787829040.999405:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787829040.999408:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d25680 x1874670646790272/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829065 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787829040.999413:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829040.999414:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787829040.999416:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829040.999417:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787829040.999418:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829040.999420:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787829040.999422:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787829040.999424:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829040.999426:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:2.0:1787829040.999427:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1787829040.999430:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d25680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646790272:192.168.203.143@tcp:38:kworker.0 00000100:00000001:2.0:1787829040.999434:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1787829040.999436:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787829040.999437:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829040.999440:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787829040.999442:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787829040.999444:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787829040.999449:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d26940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646790400:192.168.203.143@tcp:8:kworker.0 00000100:00000001:2.0:1787829040.999456:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787829040.999458:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1787829040.999460:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829040.999462:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1787829040.999466:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829040.999472:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787829040.999473:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601346880 : -126234108204736 : ffff8d30d4d26940) 00000100:00000001:2.0:1787829040.999477:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1787829040.999493:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:2.0:1787829040.999499:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1787829040.999503:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787829040.999505:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829040.999508:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1787829040.999513:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829040.999517:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298513955 00000100:00000001:2.0:1787829040.999520:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298515200 00000100:00000001:2.0:1787829040.999522:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298513955 00000100:00000001:2.0:1787829040.999524:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298515200 00000100:00000001:2.0:1787829040.999527:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787829040.999530:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829040.999532:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1787829040.999537:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829040/real 1787829040] req@ffff8d30d4d26940 x1874670646790400/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829065 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1787829040.999545:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787829040.999547:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1787829040.999550:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1787829040.999552:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787829040.999553:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829040.999555:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787829040.999556:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787829040.999560:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d26940 x1874670646790400/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829065 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787829040.999572:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829040.999574:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787829040.999577:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829040.999578:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787829040.999579:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829040.999581:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787829040.999583:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787829040.999585:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829040.999588:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:2.0:1787829040.999590:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1787829040.999593:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d26940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646790400:192.168.203.143@tcp:8:kworker.0 00000100:00000001:2.0:1787829040.999597:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829040.999629:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787829040.999631:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787829040.999633:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1787829040.999636:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787829040.999639:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:2.0:1787829040.999640:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:2.0:1787829040.999643:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787829040.999645:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1787829040.999648:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787829040.999650:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829040.999652:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787829040.999654:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787829040.999656:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1787829040.999658:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787829040.999659:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:2.0:1787829040.999661:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:2.0:1787829040.999663:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787829040.999665:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1787829040.999667:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787829040.999669:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829040.999671:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787829040.999673:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787829040.999675:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1787829040.999677:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787829040.999678:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:2.0:1787829040.999680:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:2.0:1787829040.999683:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787829040.999684:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1787829040.999687:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787829040.999688:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829040.999690:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829040.999694:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1787829040.999696:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829040.999698:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787829040.999700:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829041.894872:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829041.894877:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829041.894953:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829041.894958:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829041.894961:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829042.918674:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829042.918681:0:815:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1787829042.918683:0:815:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1787829042.918685:0:815:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829042.918689:0:815:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1787829042.918691:0:815:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829042.918693:0:815:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829042.918695:0:815:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829042.918747:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829042.918752:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829042.918756:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829043.942708:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829043.942713:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829043.942773:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829043.942778:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829043.942782:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829044.966711:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829044.966717:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829044.966771:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829044.966774:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829044.966777:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829045.990711:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829045.990719:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829045.990798:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829045.990802:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829045.990806:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787829046.118740:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829046.118749:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1787829046.118752:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787829046.118755:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787829046.118759:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787829046.118768:0:468:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 3846 00000100:00080000:3.0:1787829046.118791:0:468:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787829046.118794:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829046.118796:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:3.0:1787829046.118800:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1787829046.118802:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1787829046.118804:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:3.0:1787829046.118806:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:3.0:1787829046.118807:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829046.118808:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:3.0:1787829046.118812:0:468:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:3.0:1787829046.118813:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829046.118815:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787829046.118819:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829046.118821:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829046.118823:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:3.0:1787829046.118825:0:468:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829046.118834:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787829046.118836:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829046.118838:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787829046.118841:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787829046.118845:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829046.118854:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829046.118861:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c7016d00 x1874670646790528/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787829046.118868:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787829046.118872:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829046.118887:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787829046.118889:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829046.118891:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1787829046.118893:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787829046.118897:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787829046.118899:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787829046.118901:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3846 00000100:00080000:3.0:1787829046.118903:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787829046.118905:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829046.118906:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:3.0:1787829046.118908:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1787829046.118909:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1787829046.118910:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000020:00000001:3.0:1787829046.118911:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:3.0:1787829046.118912:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829046.118913:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:3.0:1787829046.118916:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:3.0:1787829046.118917:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829046.118919:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787829046.118923:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829046.118925:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829046.118927:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:2.0:1787829046.118930:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000008:00100000:3.0:1787829046.118931:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1787829046.118935:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829046.118938:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787829046.118938:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829046.118943:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 02000000:00000001:3.0:1787829046.118944:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787829046.118946:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829046.118949:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787829046.118949:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787829046.118953:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:2.0:1787829046.118955:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829046.118955:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829046.118963:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829046.118964:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000100:00100000:3.0:1787829046.118969:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c70170c0 x1874670646790656/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:2.0:1787829046.118970:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787829046.118976:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829046.118981:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787829046.118984:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787829046.118988:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c7016d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646790528:192.168.203.143@tcp:250:kworker.0 00000100:00000001:3.0:1787829046.118989:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787829046.118993:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829046.118999:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1787829046.119000:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:3.0:1787829046.119001:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787829046.119003:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787829046.119006:0:468:0:(import.c:485:import_select_connection()) Process entered 02000000:00000001:2.0:1787829046.119009:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000100:00080000:3.0:1787829046.119009:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3846 00000100:00080000:3.0:1787829046.119014:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 02000000:00000001:2.0:1787829046.119015:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829046.119015:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829046.119018:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:3.0:1787829046.119021:0:468:0:(genops.c:792:class_conn2export()) Process entered 02000000:00000001:2.0:1787829046.119022:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000020:00000001:3.0:1787829046.119022:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1787829046.119026:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000020:00000001:3.0:1787829046.119028:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:3.0:1787829046.119029:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 02000000:00000001:2.0:1787829046.119030:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829046.119030:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:3.0:1787829046.119037:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:3.0:1787829046.119039:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829046.119040:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787829046.119041:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829046.119043:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829046.119045:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:3.0:1787829046.119047:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00000001:2.0:1787829046.119050:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000008:00000001:3.0:1787829046.119050:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829046.119054:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369547008 : -126234340004608 : ffff8d30c7016d00) 00000100:00000001:3.0:1787829046.119055:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829046.119063:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 02000000:00000001:3.0:1787829046.119065:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787829046.119068:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829046.119070:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787829046.119072:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787829046.119073:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829046.119080:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829046.119085:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c7017840 x1874670646790784/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787829046.119094:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787829046.119096:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829046.119099:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787829046.119102:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:2.0:1787829046.119104:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00080000:3.0:1787829046.119110:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1787829046.119112:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787829046.119113:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787829046.119116:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:2.0:1787829046.119120:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00080000:3.0:1787829046.119120:0:468:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3846 00000100:00000001:2.0:1787829046.119126:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00080000:3.0:1787829046.119126:0:468:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787829046.119131:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829046.119131:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829046.119134:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:2.0:1787829046.119135:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000020:00000001:3.0:1787829046.119139:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1787829046.119141:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:2.0:1787829046.119142:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829046.119143:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000020:00000001:3.0:1787829046.119147:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:2.0:1787829046.119148:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4298520320 00000100:00000001:3.0:1787829046.119149:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829046.119152:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:3.0:1787829046.119156:0:468:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787829046.119159:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 00000100:00000001:3.0:1787829046.119159:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829046.119162:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787829046.119166:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787829046.119168:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4298520320 02000000:00000001:3.0:1787829046.119168:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829046.119172:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:3.0:1787829046.119173:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1787829046.119175:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829046.119176:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829046.119180:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787829046.119181:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829046.119183:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:2.0:1787829046.119184:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 02000000:00000001:3.0:1787829046.119185:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787829046.119187:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829046.119191:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829046.119194:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829046.119196:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829046.119200:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c7014000 x1874670646790912/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787829046.119203:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00000001:3.0:1787829046.119208:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787829046.119211:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829046.119213:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787829046.119215:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829046/real 1787829046] req@ffff8d30c7016d00 x1874670646790528/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829071 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1787829046.119229:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787829046.119231:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1787829046.119236:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1787829046.119238:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787829046.119239:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829046.119241:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787829046.119243:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787829046.119247:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c7016d00 x1874670646790528/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829071 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787829046.119252:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829046.119254:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787829046.119256:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829046.119258:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787829046.119259:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829046.119261:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787829046.119262:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787829046.119265:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829046.119267:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:2.0:1787829046.119269:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1787829046.119271:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c7016d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646790528:192.168.203.143@tcp:250:kworker.0 00000100:00000001:2.0:1787829046.119275:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829046.119276:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787829046.119277:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787829046.119279:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1787829046.119281:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787829046.119283:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:2.0:1787829046.119284:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:2.0:1787829046.119288:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787829046.119290:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1787829046.119293:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787829046.119294:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829046.119297:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829046.119304:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1787829046.119306:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829046.119308:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787829046.119310:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1787829046.119311:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1787829046.119313:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787829046.119314:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829046.119317:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787829046.119318:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787829046.119321:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787829046.119325:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c70170c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646790656:192.168.203.143@tcp:38:kworker.0 00000100:00000001:2.0:1787829046.119329:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787829046.119331:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1787829046.119333:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829046.119335:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1787829046.119338:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829046.119344:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787829046.119345:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369547968 : -126234340003648 : ffff8d30c70170c0) 00000100:00000001:2.0:1787829046.119349:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1787829046.119371:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:2.0:1787829046.119376:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1787829046.119379:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787829046.119380:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829046.119382:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1787829046.119391:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829046.119396:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298519062 00000100:00000001:2.0:1787829046.119399:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298520320 00000100:00000001:2.0:1787829046.119402:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298519062 00000100:00000001:2.0:1787829046.119404:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298520320 00000100:00000001:2.0:1787829046.119406:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787829046.119408:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829046.119410:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1787829046.119414:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829046/real 1787829046] req@ffff8d30c70170c0 x1874670646790656/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829071 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1787829046.119420:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787829046.119421:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1787829046.119424:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1787829046.119425:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787829046.119426:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829046.119427:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787829046.119428:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787829046.119431:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c70170c0 x1874670646790656/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829071 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787829046.119435:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829046.119437:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787829046.119439:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829046.119440:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787829046.119441:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829046.119442:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787829046.119443:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787829046.119446:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829046.119448:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:2.0:1787829046.119449:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1787829046.119451:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c70170c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646790656:192.168.203.143@tcp:38:kworker.0 00000100:00000001:2.0:1787829046.119453:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1787829046.119454:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787829046.119455:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829046.119456:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787829046.119457:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787829046.119459:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787829046.119460:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c7017840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646790784:192.168.203.143@tcp:38:kworker.0 00000100:00000001:2.0:1787829046.119463:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787829046.119464:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1787829046.119466:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829046.119467:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1787829046.119469:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829046.119472:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787829046.119473:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369549888 : -126234340001728 : ffff8d30c7017840) 00000100:00000001:2.0:1787829046.119475:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1787829046.119485:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:2.0:1787829046.119488:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1787829046.119489:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787829046.119490:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829046.119491:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1787829046.119493:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829046.119495:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298519062 00000100:00000001:2.0:1787829046.119497:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298520320 00000100:00000001:2.0:1787829046.119499:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298519062 00000100:00000001:2.0:1787829046.119501:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298520320 00000100:00000001:2.0:1787829046.119502:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787829046.119504:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829046.119505:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1787829046.119507:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829046/real 1787829046] req@ffff8d30c7017840 x1874670646790784/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829071 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1787829046.119512:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787829046.119513:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1787829046.119515:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1787829046.119516:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787829046.119517:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829046.119519:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787829046.119521:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787829046.119525:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c7017840 x1874670646790784/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829071 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787829046.119532:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829046.119534:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787829046.119536:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829046.119538:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787829046.119539:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829046.119541:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787829046.119543:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787829046.119545:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829046.119548:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:2.0:1787829046.119551:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1787829046.119554:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c7017840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646790784:192.168.203.143@tcp:38:kworker.0 00000100:00000001:2.0:1787829046.119559:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1787829046.119560:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787829046.119562:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829046.119564:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787829046.119566:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787829046.119568:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787829046.119571:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c7014000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646790912:192.168.203.143@tcp:8:kworker.0 00000100:00000001:2.0:1787829046.119575:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787829046.119576:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1787829046.119578:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829046.119579:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1787829046.119581:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829046.119585:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787829046.119586:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369535488 : -126234340016128 : ffff8d30c7014000) 00000100:00000001:2.0:1787829046.119589:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1787829046.119598:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:2.0:1787829046.119633:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1787829046.119634:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787829046.119635:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829046.119637:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1787829046.119638:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829046.119641:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298519076 00000100:00000001:2.0:1787829046.119643:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298520320 00000100:00000001:2.0:1787829046.119644:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298519076 00000100:00000001:2.0:1787829046.119646:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298520320 00000100:00000001:2.0:1787829046.119647:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787829046.119648:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829046.119650:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1787829046.119652:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829046/real 1787829046] req@ffff8d30c7014000 x1874670646790912/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829071 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1787829046.119657:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787829046.119658:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1787829046.119660:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1787829046.119661:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787829046.119661:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829046.119663:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787829046.119664:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787829046.119666:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c7014000 x1874670646790912/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829071 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787829046.119675:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829046.119677:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787829046.119678:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829046.119679:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787829046.119680:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829046.119681:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787829046.119682:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787829046.119684:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829046.119686:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:2.0:1787829046.119687:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1787829046.119689:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c7014000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646790912:192.168.203.143@tcp:8:kworker.0 00000100:00000001:2.0:1787829046.119691:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829046.119692:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787829046.119693:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787829046.119694:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1787829046.119695:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787829046.119696:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:2.0:1787829046.119697:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:2.0:1787829046.119699:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787829046.119700:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1787829046.119701:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787829046.119702:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829046.119703:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787829046.119704:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787829046.119705:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1787829046.119712:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787829046.119712:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:2.0:1787829046.119713:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:2.0:1787829046.119714:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787829046.119715:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1787829046.119716:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787829046.119717:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829046.119718:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787829046.119719:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787829046.119720:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1787829046.119721:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787829046.119722:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:2.0:1787829046.119723:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:2.0:1787829046.119724:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787829046.119725:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1787829046.119726:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787829046.119727:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829046.119728:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829046.119730:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1787829046.119731:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829046.119732:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787829046.119734:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829047.014715:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829047.014722:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829047.014918:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829047.014922:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829047.014926:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829048.038713:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829048.038719:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829048.038806:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829048.038812:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829048.038815:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829049.062719:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829049.062726:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829049.062789:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829049.062795:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829049.062799:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829050.086730:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829050.086737:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829050.086834:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829050.086842:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829050.086847:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829051.110678:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829051.110684:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829051.110797:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829051.110805:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829051.110811:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787829051.238702:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829051.238710:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1787829051.238712:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787829051.238714:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787829051.238718:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787829051.238723:0:468:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 3851 00000100:00080000:3.0:1787829051.238732:0:468:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787829051.238734:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829051.238735:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:3.0:1787829051.238739:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1787829051.238740:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1787829051.238755:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:3.0:1787829051.238757:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:3.0:1787829051.238758:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829051.238759:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:3.0:1787829051.238763:0:468:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:3.0:1787829051.238765:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829051.238767:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787829051.238770:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829051.238772:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829051.238774:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:3.0:1787829051.238776:0:468:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829051.238785:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787829051.238787:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829051.238789:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787829051.238791:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787829051.238800:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829051.238808:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829051.238815:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c7016940 x1874670646791040/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787829051.238822:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787829051.238825:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829051.238836:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787829051.238839:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829051.238841:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1787829051.238842:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787829051.238844:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787829051.238846:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787829051.238848:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3851 00000100:00080000:3.0:1787829051.238850:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787829051.238852:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829051.238853:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:3.0:1787829051.238854:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1787829051.238856:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1787829051.238857:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000020:00000001:3.0:1787829051.238859:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:3.0:1787829051.238860:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829051.238861:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:3.0:1787829051.238864:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:3.0:1787829051.238866:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829051.238867:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787829051.238871:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829051.238873:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829051.238874:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:2.0:1787829051.238876:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000008:00100000:3.0:1787829051.238877:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1787829051.238881:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829051.238882:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787829051.238883:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829051.238887:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 02000000:00000001:3.0:1787829051.238888:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787829051.238891:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829051.238893:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787829051.238893:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787829051.238896:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:2.0:1787829051.238898:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829051.238898:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829051.238906:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:3.0:1787829051.238907:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829051.238912:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00100000:3.0:1787829051.238913:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c7017c00 x1874670646791168/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:2.0:1787829051.238919:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829051.238925:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787829051.238929:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787829051.238931:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c7016940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646791040:192.168.203.143@tcp:250:kworker.0 00000100:00000001:3.0:1787829051.238932:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829051.238939:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00080000:3.0:1787829051.238941:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 02000000:00000001:2.0:1787829051.238943:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000100:00080000:3.0:1787829051.238945:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 02000000:00000001:2.0:1787829051.238948:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829051.238948:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787829051.238952:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 02000000:00000001:2.0:1787829051.238953:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:3.0:1787829051.238955:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787829051.238959:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3851 02000000:00000001:2.0:1787829051.238960:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787829051.238961:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787829051.238966:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829051.238967:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:3.0:1787829051.238968:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1787829051.238969:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1787829051.238970:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000020:00000001:3.0:1787829051.238972:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:3.0:1787829051.238973:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829051.238974:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:3.0:1787829051.238976:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787829051.238978:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787829051.238979:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829051.238983:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369546048 : -126234340005568 : ffff8d30c7016940) 02000000:00000001:3.0:1787829051.238983:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787829051.238985:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829051.238989:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829051.238991:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:3.0:1787829051.238991:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:3.0:1787829051.238994:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1787829051.238997:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829051.238998:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829051.239003:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787829051.239004:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829051.239005:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787829051.239006:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787829051.239007:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829051.239011:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829051.239015:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c7014000 x1874670646791296/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787829051.239023:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787829051.239026:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829051.239029:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787829051.239031:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829051.239035:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1787829051.239036:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787829051.239038:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:2.0:1787829051.239045:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:3.0:1787829051.239045:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787829051.239052:0:468:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3851 00000100:00080000:3.0:1787829051.239058:0:468:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787829051.239061:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787829051.239061:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829051.239066:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:2.0:1787829051.239067:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000020:00000001:3.0:1787829051.239069:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:2.0:1787829051.239072:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829051.239072:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1787829051.239075:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:2.0:1787829051.239076:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000020:00000001:3.0:1787829051.239079:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:3.0:1787829051.239082:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829051.239083:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:2.0:1787829051.239084:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787829051.239087:0:468:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:3.0:1787829051.239091:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829051.239092:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4298525440 02000000:00000001:3.0:1787829051.239092:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787829051.239097:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829051.239100:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829051.239101:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:2.0:1787829051.239104:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 00000008:00100000:3.0:1787829051.239104:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1787829051.239106:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829051.239109:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829051.239113:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4298525440 02000000:00000001:3.0:1787829051.239114:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787829051.239116:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829051.239119:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787829051.239120:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787829051.239121:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829051.239122:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 00000100:00000001:3.0:1787829051.239127:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829051.239129:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787829051.239133:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829051.239136:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1787829051.239141:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c7017840 x1874670646791424/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787829051.239144:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829051/real 1787829051] req@ffff8d30c7016940 x1874670646791040/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829076 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829051.239158:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787829051.239163:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829051.239166:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:2.0:1787829051.239167:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787829051.239171:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1787829051.239176:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1787829051.239178:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787829051.239180:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829051.239183:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787829051.239185:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787829051.239191:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c7016940 x1874670646791040/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829076 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787829051.239201:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829051.239204:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787829051.239208:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829051.239210:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787829051.239212:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829051.239216:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787829051.239218:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787829051.239222:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829051.239226:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:2.0:1787829051.239229:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1787829051.239233:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c7016940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646791040:192.168.203.143@tcp:250:kworker.0 00000100:00000001:2.0:1787829051.239239:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829051.239241:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787829051.239243:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787829051.239245:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1787829051.239248:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787829051.239250:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:2.0:1787829051.239253:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:2.0:1787829051.239256:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787829051.239258:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1787829051.239260:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787829051.239261:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829051.239264:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829051.239270:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1787829051.239272:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829051.239275:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787829051.239276:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1787829051.239278:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1787829051.239279:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787829051.239281:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829051.239283:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787829051.239285:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787829051.239287:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787829051.239290:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c7017c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646791168:192.168.203.143@tcp:38:kworker.0 00000100:00000001:2.0:1787829051.239294:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787829051.239296:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1787829051.239297:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829051.239299:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1787829051.239302:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829051.239308:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787829051.239309:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369550848 : -126234340000768 : ffff8d30c7017c00) 00000100:00000001:2.0:1787829051.239314:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1787829051.239326:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:2.0:1787829051.239331:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1787829051.239333:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787829051.239335:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829051.239337:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1787829051.239339:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829051.239343:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298524182 00000100:00000001:2.0:1787829051.239346:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298525440 00000100:00000001:2.0:1787829051.239349:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298524182 00000100:00000001:2.0:1787829051.239351:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298525440 00000100:00000001:2.0:1787829051.239353:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787829051.239355:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829051.239357:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1787829051.239361:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829051/real 1787829051] req@ffff8d30c7017c00 x1874670646791168/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829076 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1787829051.239370:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787829051.239371:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1787829051.239373:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1787829051.239375:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787829051.239377:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829051.239378:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787829051.239380:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787829051.239384:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c7017c00 x1874670646791168/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829076 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787829051.239392:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829051.239394:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787829051.239397:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829051.239398:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787829051.239400:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829051.239402:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787829051.239403:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787829051.239406:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829051.239409:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:2.0:1787829051.239411:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1787829051.239415:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c7017c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646791168:192.168.203.143@tcp:38:kworker.0 00000100:00000001:2.0:1787829051.239418:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1787829051.239420:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787829051.239421:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829051.239423:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787829051.239425:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787829051.239427:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787829051.239430:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c7014000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646791296:192.168.203.143@tcp:38:kworker.0 00000100:00000001:2.0:1787829051.239433:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787829051.239435:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1787829051.239437:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829051.239439:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1787829051.239441:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829051.239446:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787829051.239447:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369535488 : -126234340016128 : ffff8d30c7014000) 00000100:00000001:2.0:1787829051.239451:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1787829051.239461:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:2.0:1787829051.239465:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1787829051.239467:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787829051.239469:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829051.239471:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1787829051.239473:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829051.239476:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298524182 00000100:00000001:2.0:1787829051.239478:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298525440 00000100:00000001:2.0:1787829051.239481:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298524182 00000100:00000001:2.0:1787829051.239483:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298525440 00000100:00000001:2.0:1787829051.239486:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787829051.239488:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829051.239489:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1787829051.239493:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829051/real 1787829051] req@ffff8d30c7014000 x1874670646791296/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829076 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1787829051.239502:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787829051.239503:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1787829051.239505:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1787829051.239507:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787829051.239509:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829051.239510:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787829051.239512:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787829051.239515:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c7014000 x1874670646791296/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829076 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787829051.239524:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829051.239526:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787829051.239529:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829051.239531:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787829051.239532:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829051.239534:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787829051.239535:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787829051.239538:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829051.239541:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:2.0:1787829051.239543:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1787829051.239546:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c7014000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646791296:192.168.203.143@tcp:38:kworker.0 00000100:00000001:2.0:1787829051.239550:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1787829051.239551:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787829051.239553:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829051.239555:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787829051.239557:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787829051.239559:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787829051.239562:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c7017840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646791424:192.168.203.143@tcp:8:kworker.0 00000100:00000001:2.0:1787829051.239566:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787829051.239568:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1787829051.239569:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829051.239571:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1787829051.239574:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829051.239578:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787829051.239580:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369549888 : -126234340001728 : ffff8d30c7017840) 00000100:00000001:2.0:1787829051.239583:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1787829051.239592:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:2.0:1787829051.239596:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1787829051.239598:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787829051.239624:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829051.239626:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1787829051.239629:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829051.239633:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298524196 00000100:00000001:2.0:1787829051.239635:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298525440 00000100:00000001:2.0:1787829051.239638:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298524196 00000100:00000001:2.0:1787829051.239640:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298525440 00000100:00000001:2.0:1787829051.239642:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787829051.239644:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829051.239646:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1787829051.239651:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829051/real 1787829051] req@ffff8d30c7017840 x1874670646791424/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829076 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1787829051.239660:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787829051.239661:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1787829051.239663:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1787829051.239665:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787829051.239666:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829051.239668:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787829051.239670:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787829051.239674:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c7017840 x1874670646791424/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829076 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787829051.239682:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829051.239684:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787829051.239687:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829051.239689:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787829051.239690:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829051.239692:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787829051.239693:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787829051.239696:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829051.239699:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:2.0:1787829051.239702:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1787829051.239705:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c7017840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646791424:192.168.203.143@tcp:8:kworker.0 00000100:00000001:2.0:1787829051.239709:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829051.239711:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787829051.239712:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787829051.239713:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1787829051.239715:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787829051.239717:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:2.0:1787829051.239718:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:2.0:1787829051.239720:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787829051.239722:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1787829051.239724:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787829051.239725:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829051.239727:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787829051.239728:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787829051.239729:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1787829051.239731:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787829051.239732:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:2.0:1787829051.239733:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:2.0:1787829051.239735:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787829051.239736:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1787829051.239738:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787829051.239739:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829051.239754:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787829051.239756:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787829051.239757:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1787829051.239759:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787829051.239760:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:2.0:1787829051.239761:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:2.0:1787829051.239763:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787829051.239764:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1787829051.239766:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787829051.239767:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829051.239769:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829051.239772:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1787829051.239774:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829051.239776:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787829051.239778:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829052.134724:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829052.134730:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829052.134803:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829052.134806:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829052.134809:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829053.158674:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829053.158680:0:815:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1787829053.158681:0:815:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1787829053.158684:0:815:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829053.158688:0:815:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1787829053.158690:0:815:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829053.158691:0:815:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829053.158693:0:815:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829053.158751:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829053.158754:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829053.158757:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829054.182694:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829054.182699:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829054.182752:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829054.182757:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829054.182761:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829055.206735:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829055.206740:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829055.206798:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829055.206802:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829055.206805:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829056.230751:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829056.230759:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829056.230817:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829056.230820:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829056.230823:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787829056.358766:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829056.358774:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1787829056.358776:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787829056.358779:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787829056.358783:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787829056.358789:0:40798:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 3856 00000100:00080000:0.0:1787829056.358798:0:40798:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787829056.358801:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829056.358802:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787829056.358805:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787829056.358807:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787829056.358811:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:0.0:1787829056.358812:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:0.0:1787829056.358814:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829056.358815:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:0.0:1787829056.358818:0:40798:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787829056.358820:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829056.358822:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787829056.358825:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787829056.358827:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829056.358829:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1787829056.358831:0:40798:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829056.358846:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787829056.358847:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829056.358850:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787829056.358853:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787829056.358856:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829056.358867:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787829056.358874:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d26940 x1874670646791552/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787829056.358882:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787829056.358886:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829056.358901:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787829056.358903:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829056.358905:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1787829056.358906:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787829056.358908:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787829056.358909:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787829056.358911:0:40798:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3856 00000100:00080000:0.0:1787829056.358913:0:40798:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787829056.358915:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829056.358915:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787829056.358917:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787829056.358919:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787829056.358921:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000020:00000001:0.0:1787829056.358926:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:0.0:1787829056.358929:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829056.358929:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:0.0:1787829056.358933:0:40798:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787829056.358935:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829056.358937:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:2.0:1787829056.358940:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:0.0:1787829056.358941:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787829056.358945:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829056.358945:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787829056.358946:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:2.0:1787829056.358949:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000008:00100000:0.0:1787829056.358951:0:40798:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00000001:2.0:1787829056.358952:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787829056.358955:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:0.0:1787829056.358956:0:40798:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829056.358959:0:40798:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829056.358961:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787829056.358963:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787829056.358965:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787829056.358966:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829056.358971:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829056.358972:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787829056.358976:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00100000:2.0:1787829056.358978:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d26940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646791552:192.168.203.143@tcp:250:kworker.0 02000000:00000001:0.0:1787829056.358980:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829056.358986:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787829056.358990:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:0.0:1787829056.358991:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829056.358992:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829056.358996:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1787829056.359000:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787829056.359002:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d25680 x1874670646791680/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787829056.359013:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787829056.359014:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787829056.359016:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829056.359017:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601346880 : -126234108204736 : ffff8d30d4d26940) 00000100:00000001:2.0:1787829056.359024:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:0.0:1787829056.359025:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787829056.359030:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829056.359033:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1787829056.359034:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787829056.359035:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787829056.359038:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787829056.359040:0:40798:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3856 00000100:00080000:0.0:1787829056.359044:0:40798:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787829056.359046:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829056.359047:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787829056.359049:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787829056.359050:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787829056.359052:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000020:00000001:0.0:1787829056.359053:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:0.0:1787829056.359054:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829056.359056:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:0.0:1787829056.359059:0:40798:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787829056.359063:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829056.359066:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000800:00000100:2.0:1787829056.359066:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 02000000:00000001:0.0:1787829056.359070:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787829056.359072:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829056.359074:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1787829056.359076:0:40798:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1787829056.359078:0:40798:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829056.359080:0:40798:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829056.359083:0:2390:0:(events.c:34:request_out_callback()) Process entered 02000000:00000001:0.0:1787829056.359084:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787829056.359087:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829056.359088:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:2.0:1787829056.359089:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 02000000:00000001:0.0:1787829056.359090:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787829056.359093:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829056.359095:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829056.359098:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787829056.359103:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d252c0 x1874670646791808/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787829056.359108:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787829056.359109:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787829056.359113:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829056.359115:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787829056.359116:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:2.0:1787829056.359118:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787829056.359120:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1787829056.359123:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787829056.359124:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787829056.359125:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4298530560 00000100:00000001:0.0:1787829056.359128:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787829056.359131:0:40798:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3856 00000100:00080000:0.0:1787829056.359133:0:40798:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787829056.359135:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829056.359135:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787829056.359137:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787829056.359138:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:2.0:1787829056.359138:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 00000020:00000001:0.0:1787829056.359139:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000020:00000001:0.0:1787829056.359143:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:0.0:1787829056.359144:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829056.359145:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:2.0:1787829056.359147:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4298530560 00000100:00080000:0.0:1787829056.359149:0:40798:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787829056.359152:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1787829056.359156:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829056.359157:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787829056.359162:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829056.359164:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1787829056.359171:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829056/real 1787829056] req@ffff8d30d4d26940 x1874670646791552/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829081 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:0.0:1787829056.359173:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787829056.359178:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787829056.359184:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829056.359188:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1787829056.359193:0:40798:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000400:00000001:2.0:1787829056.359193:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787829056.359197:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000008:00000001:0.0:1787829056.359198:0:40798:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829056.359202:0:40798:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829056.359203:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1787829056.359207:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787829056.359212:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:0.0:1787829056.359214:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:2.0:1787829056.359215:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 02000000:00000001:0.0:1787829056.359217:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829056.359219:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829056.359220:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787829056.359222:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787829056.359226:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829056.359230:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787829056.359230:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d26940 x1874670646791552/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829081 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829056.359239:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d27c00 x1874670646791936/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787829056.359242:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829056.359247:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787829056.359250:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787829056.359252:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829056.359254:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829056.359255:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787829056.359258:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829056.359259:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829056.359261:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787829056.359265:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787829056.359268:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829056.359271:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:2.0:1787829056.359273:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1787829056.359276:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d26940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646791552:192.168.203.143@tcp:250:kworker.0 00000100:00000001:2.0:1787829056.359281:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829056.359283:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787829056.359284:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787829056.359286:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1787829056.359290:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787829056.359291:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:2.0:1787829056.359293:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:2.0:1787829056.359297:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787829056.359298:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1787829056.359301:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787829056.359302:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829056.359303:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829056.359309:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1787829056.359310:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829056.359312:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787829056.359313:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1787829056.359314:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1787829056.359315:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787829056.359316:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829056.359318:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787829056.359319:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787829056.359320:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787829056.359323:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d25680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646791680:192.168.203.143@tcp:38:kworker.0 00000100:00000001:2.0:1787829056.359325:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787829056.359327:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1787829056.359328:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829056.359329:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1787829056.359331:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829056.359336:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787829056.359337:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601342080 : -126234108209536 : ffff8d30d4d25680) 00000100:00000001:2.0:1787829056.359341:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1787829056.359351:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:2.0:1787829056.359355:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1787829056.359357:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787829056.359358:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829056.359359:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1787829056.359361:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829056.359364:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298529302 00000100:00000001:2.0:1787829056.359366:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298530560 00000100:00000001:2.0:1787829056.359367:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298529302 00000100:00000001:2.0:1787829056.359369:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298530560 00000100:00000001:2.0:1787829056.359370:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787829056.359372:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829056.359373:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1787829056.359375:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829056/real 1787829056] req@ffff8d30d4d25680 x1874670646791680/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829081 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1787829056.359390:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787829056.359391:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1787829056.359393:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1787829056.359394:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787829056.359395:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829056.359397:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787829056.359398:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787829056.359400:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d25680 x1874670646791680/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829081 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787829056.359405:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829056.359407:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787829056.359409:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829056.359410:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787829056.359410:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829056.359412:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787829056.359413:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787829056.359415:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829056.359417:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:2.0:1787829056.359418:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1787829056.359420:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d25680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646791680:192.168.203.143@tcp:38:kworker.0 00000100:00000001:2.0:1787829056.359423:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1787829056.359424:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787829056.359425:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829056.359426:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787829056.359427:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787829056.359429:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787829056.359432:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d252c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646791808:192.168.203.143@tcp:38:kworker.0 00000100:00000001:2.0:1787829056.359435:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787829056.359437:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1787829056.359438:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829056.359440:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1787829056.359442:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829056.359447:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787829056.359448:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601341120 : -126234108210496 : ffff8d30d4d252c0) 00000100:00000001:2.0:1787829056.359452:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1787829056.359465:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:2.0:1787829056.359470:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1787829056.359472:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787829056.359474:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829056.359476:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1787829056.359479:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829056.359482:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298529302 00000100:00000001:2.0:1787829056.359485:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298530560 00000100:00000001:2.0:1787829056.359486:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298529302 00000100:00000001:2.0:1787829056.359488:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298530560 00000100:00000001:2.0:1787829056.359489:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787829056.359491:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829056.359493:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1787829056.359496:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829056/real 1787829056] req@ffff8d30d4d252c0 x1874670646791808/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829081 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1787829056.359502:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787829056.359503:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1787829056.359505:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1787829056.359506:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787829056.359507:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829056.359508:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787829056.359509:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787829056.359511:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d252c0 x1874670646791808/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829081 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787829056.359516:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829056.359517:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787829056.359519:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829056.359520:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787829056.359520:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829056.359521:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787829056.359522:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787829056.359524:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829056.359526:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:2.0:1787829056.359527:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1787829056.359529:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d252c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646791808:192.168.203.143@tcp:38:kworker.0 00000100:00000001:2.0:1787829056.359532:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1787829056.359532:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787829056.359533:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829056.359535:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787829056.359536:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787829056.359537:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787829056.359539:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d27c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646791936:192.168.203.143@tcp:8:kworker.0 00000100:00000001:2.0:1787829056.359543:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787829056.359544:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1787829056.359546:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829056.359547:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1787829056.359549:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829056.359552:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787829056.359553:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601351680 : -126234108199936 : ffff8d30d4d27c00) 00000100:00000001:2.0:1787829056.359556:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1787829056.359564:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:2.0:1787829056.359567:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1787829056.359568:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787829056.359569:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829056.359570:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1787829056.359572:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829056.359574:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298529315 00000100:00000001:2.0:1787829056.359575:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298530560 00000100:00000001:2.0:1787829056.359577:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298529315 00000100:00000001:2.0:1787829056.359578:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298530560 00000100:00000001:2.0:1787829056.359580:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787829056.359581:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829056.359582:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1787829056.359584:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829056/real 1787829056] req@ffff8d30d4d27c00 x1874670646791936/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829081 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1787829056.359589:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787829056.359590:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1787829056.359592:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1787829056.359593:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787829056.359594:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829056.359595:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787829056.359595:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787829056.359598:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d27c00 x1874670646791936/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829081 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787829056.359642:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829056.359643:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787829056.359645:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829056.359646:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787829056.359646:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829056.359648:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787829056.359649:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787829056.359651:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829056.359653:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:2.0:1787829056.359654:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1787829056.359656:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d27c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646791936:192.168.203.143@tcp:8:kworker.0 00000100:00000001:2.0:1787829056.359659:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829056.359660:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787829056.359661:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787829056.359662:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1787829056.359664:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787829056.359665:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:2.0:1787829056.359666:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:2.0:1787829056.359668:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787829056.359669:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1787829056.359671:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787829056.359672:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829056.359673:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787829056.359674:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787829056.359675:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1787829056.359676:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787829056.359677:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:2.0:1787829056.359677:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:2.0:1787829056.359678:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787829056.359679:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1787829056.359680:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787829056.359680:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829056.359681:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787829056.359682:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787829056.359683:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1787829056.359684:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787829056.359685:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:2.0:1787829056.359686:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:2.0:1787829056.359687:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787829056.359688:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1787829056.359690:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787829056.359691:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829056.359692:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829056.359694:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1787829056.359695:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829056.359696:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787829056.359697:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829057.254691:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829057.254696:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829057.254769:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829057.254773:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829057.254777:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829058.278786:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829058.278796:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829058.278867:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829058.278872:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829058.278877:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829059.302678:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829059.302697:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829059.302752:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829059.302756:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829059.302759:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829060.326710:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829060.326715:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829060.326769:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829060.326772:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829060.326774:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829061.350695:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829061.350702:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829061.350760:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829061.350763:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829061.350766:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787829061.478704:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829061.478712:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1787829061.478715:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787829061.478719:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787829061.478724:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787829061.478730:0:40798:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 3861 00000100:00080000:2.0:1787829061.478741:0:40798:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787829061.478744:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829061.478746:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787829061.478750:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787829061.478753:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787829061.478756:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:2.0:1787829061.478759:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:2.0:1787829061.478762:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829061.478764:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:2.0:1787829061.478769:0:40798:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787829061.478771:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829061.478774:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787829061.478777:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787829061.478779:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829061.478780:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:2.0:1787829061.478782:0:40798:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829061.478792:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787829061.478793:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829061.478796:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787829061.478798:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787829061.478802:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829061.478811:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787829061.478821:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2c3c0 x1874670646792064/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787829061.478832:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787829061.478836:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829061.478855:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787829061.478859:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829061.478862:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1787829061.478863:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787829061.478864:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787829061.478866:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787829061.478868:0:40798:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3861 00000100:00080000:2.0:1787829061.478872:0:40798:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787829061.478874:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829061.478875:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787829061.478877:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787829061.478879:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787829061.478880:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000020:00000001:2.0:1787829061.478882:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:0.0:1787829061.478885:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787829061.478885:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829061.478887:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:0.0:1787829061.478891:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00080000:2.0:1787829061.478891:0:40798:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787829061.478895:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1787829061.478896:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829061.478900:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 02000000:00000001:2.0:1787829061.478900:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:0.0:1787829061.478904:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829061.478907:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787829061.478910:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787829061.478911:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829061.478914:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787829061.478914:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 02000000:00000001:0.0:1787829061.478918:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000008:00100000:2.0:1787829061.478919:0:40798:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1787829061.478924:0:40798:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787829061.478925:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646792064:192.168.203.143@tcp:250:kworker.0 00000100:00000001:2.0:1787829061.478926:0:40798:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829061.478932:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787829061.478936:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787829061.478938:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829061.478940:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787829061.478941:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1787829061.478942:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829061.478945:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787829061.478946:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787829061.478947:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829061.478949:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829061.478956:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787829061.478961:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2fc00 x1874670646792192/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787829061.478962:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787829061.478965:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417869248 : -126234291682368 : ffff8d30c9e2c3c0) 00000100:00000001:0.0:1787829061.478971:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:2.0:1787829061.478973:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787829061.478975:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829061.478979:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787829061.478982:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829061.478985:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1787829061.478987:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787829061.478988:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787829061.478991:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787829061.478995:0:40798:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3861 00000100:00080000:2.0:1787829061.479000:0:40798:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787829061.479002:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829061.479006:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000800:00000100:0.0:1787829061.479007:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000020:00000001:2.0:1787829061.479012:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787829061.479014:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787829061.479017:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:0.0:1787829061.479018:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000020:00000001:2.0:1787829061.479021:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:0.0:1787829061.479025:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787829061.479025:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829061.479027:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:0.0:1787829061.479030:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787829061.479035:0:40798:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787829061.479036:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1787829061.479041:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829061.479044:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:0.0:1787829061.479045:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829061.479048:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787829061.479051:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4298535680 02000000:00000001:2.0:1787829061.479053:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829061.479058:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1787829061.479060:0:40798:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1787829061.479064:0:40798:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829061.479065:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 00000100:00000001:2.0:1787829061.479066:0:40798:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829061.479074:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:0.0:1787829061.479075:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4298535680 02000000:00000001:2.0:1787829061.479076:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829061.479082:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:0.0:1787829061.479083:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 02000000:00000001:2.0:1787829061.479084:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:0.0:1787829061.479088:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787829061.479088:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829061.479092:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829061.479095:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00000001:2.0:1787829061.479098:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787829061.479101:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829061/real 1787829061] req@ffff8d30c9e2c3c0 x1874670646792064/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829086 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829061.479108:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2d2c0 x1874670646792320/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000001:0.0:1787829061.479113:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787829061.479115:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787829061.479120:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1787829061.479121:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787829061.479125:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829061.479127:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787829061.479128:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829061.479133:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:2.0:1787829061.479133:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829061.479140:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1787829061.479141:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787829061.479142:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787829061.479144:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787829061.479146:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829061.479148:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787829061.479152:0:40798:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3861 00000100:00080000:2.0:1787829061.479156:0:40798:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1787829061.479157:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2c3c0 x1874670646792064/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829086 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787829061.479158:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829061.479161:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787829061.479165:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787829061.479167:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787829061.479169:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000020:00000001:2.0:1787829061.479174:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:0.0:1787829061.479179:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829061.479183:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787829061.479185:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829061.479186:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829061.479189:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787829061.479190:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:0.0:1787829061.479192:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:2.0:1787829061.479196:0:40798:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787829061.479198:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787829061.479200:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829061.479201:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 02000000:00000001:2.0:1787829061.479203:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00080000:0.0:1787829061.479207:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 02000000:00000001:2.0:1787829061.479208:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787829061.479214:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787829061.479217:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:2.0:1787829061.479217:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1787829061.479219:0:40798:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00000001:0.0:1787829061.479221:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000008:00000001:2.0:1787829061.479223:0:40798:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829061.479226:0:40798:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787829061.479230:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646792064:192.168.203.143@tcp:250:kworker.0 02000000:00000001:2.0:1787829061.479231:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787829061.479234:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829061.479237:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787829061.479239:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787829061.479242:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829061.479244:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829061.479246:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829061.479252:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00100000:2.0:1787829061.479254:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2cb40 x1874670646792448/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787829061.479256:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787829061.479261:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 00000100:00000001:2.0:1787829061.479262:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787829061.479265:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829061.479267:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829061.479268:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787829061.479272:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787829061.479275:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787829061.479280:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787829061.479282:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787829061.479287:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787829061.479289:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829061.479292:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829061.479299:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787829061.479301:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829061.479304:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787829061.479306:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787829061.479308:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1787829061.479309:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787829061.479311:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829061.479314:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787829061.479316:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787829061.479318:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787829061.479323:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646792192:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787829061.479327:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787829061.479329:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787829061.479331:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829061.479333:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1787829061.479336:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829061.479343:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787829061.479344:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417883648 : -126234291667968 : ffff8d30c9e2fc00) 00000100:00000001:0.0:1787829061.479349:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1787829061.479360:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:0.0:1787829061.479364:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1787829061.479366:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787829061.479367:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829061.479368:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787829061.479371:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829061.479374:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298534422 00000100:00000001:0.0:1787829061.479376:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298535680 00000100:00000001:0.0:1787829061.479378:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298534422 00000100:00000001:0.0:1787829061.479380:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298535680 00000100:00000001:0.0:1787829061.479381:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787829061.479382:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829061.479384:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787829061.479387:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829061/real 1787829061] req@ffff8d30c9e2fc00 x1874670646792192/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829086 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787829061.479392:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787829061.479393:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787829061.479394:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787829061.479395:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787829061.479396:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829061.479404:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787829061.479405:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787829061.479407:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2fc00 x1874670646792192/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829086 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787829061.479412:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829061.479414:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787829061.479415:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829061.479416:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787829061.479417:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829061.479418:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787829061.479420:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787829061.479421:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829061.479424:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787829061.479425:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787829061.479427:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646792192:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787829061.479429:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1787829061.479430:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787829061.479431:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829061.479432:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787829061.479433:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787829061.479435:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787829061.479437:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2d2c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646792320:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787829061.479439:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787829061.479440:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787829061.479442:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829061.479443:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1787829061.479445:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829061.479447:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787829061.479448:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417873088 : -126234291678528 : ffff8d30c9e2d2c0) 00000100:00000001:0.0:1787829061.479450:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1787829061.479458:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:0.0:1787829061.479461:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1787829061.479462:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787829061.479463:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829061.479464:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787829061.479466:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829061.479468:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298534422 00000100:00000001:0.0:1787829061.479469:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298535680 00000100:00000001:0.0:1787829061.479471:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298534422 00000100:00000001:0.0:1787829061.479473:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298535680 00000100:00000001:0.0:1787829061.479474:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787829061.479475:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829061.479476:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787829061.479479:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829061/real 1787829061] req@ffff8d30c9e2d2c0 x1874670646792320/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829086 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787829061.479483:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787829061.479484:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787829061.479485:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787829061.479486:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787829061.479487:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829061.479488:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787829061.479488:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787829061.479491:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2d2c0 x1874670646792320/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829086 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787829061.479495:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829061.479496:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787829061.479498:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829061.479499:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787829061.479500:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829061.479501:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787829061.479502:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787829061.479503:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829061.479506:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787829061.479507:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787829061.479509:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2d2c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646792320:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787829061.479511:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1787829061.479512:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787829061.479513:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829061.479514:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787829061.479515:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787829061.479517:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787829061.479518:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646792448:192.168.203.143@tcp:8:kworker.0 00000100:00000001:0.0:1787829061.479520:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787829061.479521:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787829061.479522:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829061.479523:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1787829061.479532:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829061.479536:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787829061.479537:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417871168 : -126234291680448 : ffff8d30c9e2cb40) 00000100:00000001:0.0:1787829061.479539:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1787829061.479545:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:0.0:1787829061.479547:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1787829061.479549:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787829061.479550:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829061.479551:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787829061.479552:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829061.479554:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298534435 00000100:00000001:0.0:1787829061.479556:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298535680 00000100:00000001:0.0:1787829061.479557:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298534435 00000100:00000001:0.0:1787829061.479559:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298535680 00000100:00000001:0.0:1787829061.479560:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787829061.479561:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829061.479562:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787829061.479565:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829061/real 1787829061] req@ffff8d30c9e2cb40 x1874670646792448/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829086 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787829061.479571:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787829061.479572:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787829061.479573:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787829061.479574:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787829061.479575:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829061.479576:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787829061.479577:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787829061.479580:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2cb40 x1874670646792448/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829086 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787829061.479584:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829061.479585:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787829061.479586:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829061.479587:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787829061.479588:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829061.479589:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787829061.479590:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787829061.479591:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829061.479593:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787829061.479594:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787829061.479596:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646792448:192.168.203.143@tcp:8:kworker.0 00000100:00000001:0.0:1787829061.479598:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829061.479599:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787829061.479618:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787829061.479619:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787829061.479621:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787829061.479622:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787829061.479624:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787829061.479625:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787829061.479626:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787829061.479629:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787829061.479630:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829061.479631:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787829061.479632:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787829061.479633:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787829061.479634:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787829061.479635:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787829061.479636:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787829061.479637:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787829061.479637:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787829061.479639:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787829061.479640:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829061.479641:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787829061.479642:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787829061.479643:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787829061.479643:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787829061.479644:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787829061.479645:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787829061.479646:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787829061.479646:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787829061.479647:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787829061.479648:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829061.479649:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829061.479651:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787829061.479652:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829061.479653:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787829061.479654:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829062.374687:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829062.374692:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829062.374736:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829062.374740:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829062.374743:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829063.398732:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829063.398741:0:815:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1787829063.398744:0:815:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1787829063.398747:0:815:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829063.398752:0:815:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1787829063.398754:0:815:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829063.398756:0:815:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829063.398758:0:815:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829063.398809:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829063.398814:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829063.398817:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829064.422715:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829064.422721:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829064.422775:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829064.422778:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829064.422781:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829065.446769:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829065.446777:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829065.446846:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829065.446851:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829065.446880:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829066.470775:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829066.470782:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829066.470845:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829066.470850:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829066.470854:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787829066.598790:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829066.598800:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1787829066.598803:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787829066.598807:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787829066.598811:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787829066.598817:0:468:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 3867 00000100:00080000:3.0:1787829066.598826:0:468:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787829066.598828:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829066.598829:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:3.0:1787829066.598833:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1787829066.598835:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1787829066.598837:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:3.0:1787829066.598839:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:3.0:1787829066.598841:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829066.598842:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:3.0:1787829066.598845:0:468:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:3.0:1787829066.598847:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829066.598848:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787829066.598851:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829066.598861:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829066.598863:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:3.0:1787829066.598865:0:468:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829066.598874:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787829066.598875:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829066.598878:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787829066.598880:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787829066.598884:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829066.598893:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829066.598900:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c70170c0 x1874670646792576/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787829066.598908:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787829066.598911:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829066.598923:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787829066.598926:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829066.598929:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1787829066.598930:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787829066.598931:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787829066.598933:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787829066.598934:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3867 00000100:00080000:3.0:1787829066.598937:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787829066.598938:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829066.598939:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:3.0:1787829066.598941:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1787829066.598942:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1787829066.598944:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000020:00000001:3.0:1787829066.598945:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:3.0:1787829066.598947:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829066.598948:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:3.0:1787829066.598951:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:3.0:1787829066.598952:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829066.598954:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:0.0:1787829066.598958:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:3.0:1787829066.598958:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829066.598962:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829066.598964:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1787829066.598965:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000008:00100000:3.0:1787829066.598967:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00000001:0.0:1787829066.598969:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000008:00000001:3.0:1787829066.598971:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829066.598973:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787829066.598974:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829066.598976:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829066.598979:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787829066.598981:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787829066.598981:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829066.598985:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787829066.598986:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829066.598987:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787829066.598992:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829066.598993:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829066.599000:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787829066.599002:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c70170c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646792576:192.168.203.143@tcp:250:kworker.0 00000100:00100000:3.0:1787829066.599007:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c7016d00 x1874670646792704/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787829066.599008:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787829066.599013:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787829066.599017:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829066.599021:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:3.0:1787829066.599026:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 02000000:00000001:0.0:1787829066.599027:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829066.599030:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829066.599034:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787829066.599036:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829066.599038:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1787829066.599040:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787829066.599041:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787829066.599042:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787829066.599043:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:0.0:1787829066.599045:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369547968 : -126234340003648 : ffff8d30c70170c0) 00000100:00080000:3.0:1787829066.599047:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3867 00000100:00000001:0.0:1787829066.599050:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00080000:3.0:1787829066.599052:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787829066.599054:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829066.599055:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:3.0:1787829066.599057:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1787829066.599058:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1787829066.599059:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000020:00000001:3.0:1787829066.599060:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:3.0:1787829066.599062:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829066.599063:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:3.0:1787829066.599066:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:3.0:1787829066.599067:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829066.599068:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787829066.599070:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829066.599071:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829066.599073:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:3.0:1787829066.599074:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1787829066.599076:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829066.599077:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829066.599081:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787829066.599083:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:0.0:1787829066.599085:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:3.0:1787829066.599086:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787829066.599089:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787829066.599091:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829066.599095:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829066.599097:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00100000:3.0:1787829066.599100:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c7014b40 x1874670646792832/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787829066.599103:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787829066.599107:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829066.599110:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1787829066.599110:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787829066.599113:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829066.599117:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829066.599117:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787829066.599119:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1787829066.599122:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4298540800 00000100:00080000:3.0:1787829066.599123:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1787829066.599127:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787829066.599128:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787829066.599130:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787829066.599132:0:468:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3867 00000100:00000001:0.0:1787829066.599133:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 00000100:00080000:3.0:1787829066.599135:0:468:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787829066.599138:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829066.599139:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:3.0:1787829066.599140:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:0.0:1787829066.599141:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4298540800 00000020:00000001:3.0:1787829066.599142:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1787829066.599145:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:0.0:1787829066.599148:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 00000020:00000001:3.0:1787829066.599148:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:3.0:1787829066.599152:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829066.599153:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:3.0:1787829066.599156:0:468:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:3.0:1787829066.599157:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829066.599158:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:0.0:1787829066.599162:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829066.599162:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829066.599164:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829066.599166:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:3.0:1787829066.599168:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00000001:0.0:1787829066.599171:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:3.0:1787829066.599172:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829066.599175:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00000001:3.0:1787829066.599175:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829066.599180:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787829066.599182:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787829066.599183:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829066/real 1787829066] req@ffff8d30c70170c0 x1874670646792576/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829091 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829066.599184:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787829066.599186:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787829066.599188:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829066.599194:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:0.0:1787829066.599201:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00100000:3.0:1787829066.599203:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c70152c0 x1874670646792960/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000001:0.0:1787829066.599204:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787829066.599208:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787829066.599211:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829066.599212:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787829066.599214:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829066.599216:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787829066.599218:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829066.599222:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787829066.599223:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c70170c0 x1874670646792576/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829091 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829066.599225:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829066.599234:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829066.599238:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787829066.599240:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829066.599242:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787829066.599243:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829066.599245:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787829066.599247:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787829066.599250:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829066.599252:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787829066.599254:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787829066.599257:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c70170c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646792576:192.168.203.143@tcp:250:kworker.0 00000100:00000001:0.0:1787829066.599260:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829066.599262:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787829066.599263:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787829066.599265:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787829066.599267:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787829066.599268:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787829066.599270:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787829066.599273:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787829066.599274:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787829066.599275:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787829066.599276:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829066.599278:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829066.599284:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787829066.599286:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829066.599287:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787829066.599288:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787829066.599290:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1787829066.599291:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787829066.599292:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829066.599293:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787829066.599294:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787829066.599296:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787829066.599298:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c7016d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646792704:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787829066.599300:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787829066.599302:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787829066.599303:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829066.599304:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1787829066.599306:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829066.599309:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787829066.599310:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369547008 : -126234340004608 : ffff8d30c7016d00) 00000100:00000001:0.0:1787829066.599314:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1787829066.599323:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:0.0:1787829066.599326:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1787829066.599328:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787829066.599329:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829066.599330:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787829066.599332:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829066.599335:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298539542 00000100:00000001:0.0:1787829066.599337:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298540800 00000100:00000001:0.0:1787829066.599339:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298539542 00000100:00000001:0.0:1787829066.599341:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298540800 00000100:00000001:0.0:1787829066.599342:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787829066.599344:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829066.599345:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787829066.599348:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829066/real 1787829066] req@ffff8d30c7016d00 x1874670646792704/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829091 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787829066.599353:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787829066.599354:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787829066.599355:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787829066.599357:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787829066.599358:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829066.599359:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787829066.599360:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787829066.599362:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c7016d00 x1874670646792704/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829091 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787829066.599367:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829066.599368:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787829066.599370:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829066.599371:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787829066.599372:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829066.599373:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787829066.599375:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787829066.599378:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829066.599381:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787829066.599384:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787829066.599387:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c7016d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646792704:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787829066.599392:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1787829066.599393:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787829066.599394:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829066.599397:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787829066.599398:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787829066.599401:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787829066.599405:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c7014b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646792832:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787829066.599409:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787829066.599411:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787829066.599412:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829066.599414:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1787829066.599417:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829066.599423:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787829066.599424:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369538368 : -126234340013248 : ffff8d30c7014b40) 00000100:00000001:0.0:1787829066.599429:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1787829066.599440:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:0.0:1787829066.599444:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1787829066.599446:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787829066.599448:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829066.599449:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787829066.599452:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829066.599455:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298539542 00000100:00000001:0.0:1787829066.599457:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298540800 00000100:00000001:0.0:1787829066.599460:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298539542 00000100:00000001:0.0:1787829066.599462:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298540800 00000100:00000001:0.0:1787829066.599464:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787829066.599466:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829066.599468:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787829066.599472:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829066/real 1787829066] req@ffff8d30c7014b40 x1874670646792832/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829091 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787829066.599481:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787829066.599482:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787829066.599485:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787829066.599486:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787829066.599488:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829066.599489:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787829066.599491:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787829066.599495:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c7014b40 x1874670646792832/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829091 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787829066.599502:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829066.599505:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787829066.599507:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829066.599509:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787829066.599510:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829066.599512:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787829066.599514:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787829066.599517:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829066.599521:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787829066.599523:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787829066.599527:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c7014b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646792832:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787829066.599531:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1787829066.599532:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787829066.599533:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829066.599536:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787829066.599538:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787829066.599540:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787829066.599543:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c70152c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646792960:192.168.203.143@tcp:8:kworker.0 00000100:00000001:0.0:1787829066.599549:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787829066.599551:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787829066.599553:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829066.599555:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1787829066.599558:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829066.599563:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787829066.599565:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369540288 : -126234340011328 : ffff8d30c70152c0) 00000100:00000001:0.0:1787829066.599568:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1787829066.599579:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:0.0:1787829066.599584:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1787829066.599586:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787829066.599588:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829066.599590:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787829066.599592:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829066.599596:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298539555 00000100:00000001:0.0:1787829066.599624:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298540800 00000100:00000001:0.0:1787829066.599626:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298539556 00000100:00000001:0.0:1787829066.599628:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298540800 00000100:00000001:0.0:1787829066.599630:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787829066.599631:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829066.599632:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787829066.599648:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829066/real 1787829066] req@ffff8d30c70152c0 x1874670646792960/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829091 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787829066.599655:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787829066.599656:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787829066.599658:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787829066.599659:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787829066.599661:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829066.599662:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787829066.599663:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787829066.599665:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c70152c0 x1874670646792960/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829091 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787829066.599675:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829066.599677:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787829066.599678:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829066.599679:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787829066.599680:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829066.599681:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787829066.599683:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787829066.599684:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829066.599686:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787829066.599687:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787829066.599689:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c70152c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646792960:192.168.203.143@tcp:8:kworker.0 00000100:00000001:0.0:1787829066.599692:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829066.599693:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787829066.599694:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787829066.599695:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787829066.599697:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787829066.599698:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787829066.599699:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787829066.599701:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787829066.599702:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787829066.599703:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787829066.599704:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829066.599705:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787829066.599706:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787829066.599707:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787829066.599708:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787829066.599709:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787829066.599710:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787829066.599711:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787829066.599712:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787829066.599713:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787829066.599713:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829066.599714:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787829066.599715:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787829066.599716:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787829066.599717:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787829066.599718:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787829066.599719:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787829066.599720:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787829066.599721:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787829066.599722:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787829066.599723:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829066.599724:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829066.599727:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787829066.599727:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829066.599729:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787829066.599730:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829067.494737:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829067.494744:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829067.494832:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829067.494837:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829067.494841:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829068.518671:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829068.518677:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829068.518722:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829068.518725:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829068.518727:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829069.542733:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829069.542740:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829069.542825:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829069.542829:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829069.542832:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829070.566678:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829070.566687:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829070.566738:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829070.566741:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829070.566743:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829071.590767:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829071.590775:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829071.590889:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829071.590894:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829071.590898:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787829071.718750:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829071.718760:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1787829071.718762:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1787829071.718783:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787829071.718788:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1787829071.718796:0:468:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 3872 00000100:00080000:1.0:1787829071.718806:0:468:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1787829071.718809:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787829071.718812:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:1.0:1787829071.718816:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:1.0:1787829071.718818:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:1.0:1787829071.718822:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:1.0:1787829071.718824:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:1.0:1787829071.718827:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787829071.718828:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:1.0:1787829071.718833:0:468:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:1.0:1787829071.718835:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829071.718838:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1787829071.718841:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829071.718845:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829071.718847:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:1.0:1787829071.718849:0:468:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829071.718860:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1787829071.718862:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829071.718864:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1787829071.718867:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1787829071.718871:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829071.718882:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829071.718891:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d13ee940 x1874670646793088/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787829071.718901:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787829071.718906:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829071.718922:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1787829071.718925:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829071.718929:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1787829071.718930:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1787829071.718932:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787829071.718935:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1787829071.718937:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3872 00000100:00080000:1.0:1787829071.718940:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1787829071.718942:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787829071.718944:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:1.0:1787829071.718946:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:1.0:1787829071.718948:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:1.0:1787829071.718951:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000020:00000001:1.0:1787829071.718953:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:1.0:1787829071.718956:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787829071.718957:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:1.0:1787829071.718962:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787829071.718963:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787829071.718965:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829071.718970:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 02000000:00000001:1.0:1787829071.718987:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:0.0:1787829071.718993:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 02000000:00000001:1.0:1787829071.718996:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829071.719000:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829071.719001:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1787829071.719005:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000008:00100000:1.0:1787829071.719006:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00000001:0.0:1787829071.719010:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:1.0:1787829071.719012:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829071.719017:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829071.719019:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787829071.719023:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787829071.719024:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829071.719027:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829071.719029:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829071.719031:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1787829071.719035:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00100000:0.0:1787829071.719039:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d13ee940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646793088:192.168.203.143@tcp:250:kworker.0 02000000:00000001:1.0:1787829071.719039:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829071.719048:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:1.0:1787829071.719048:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829071.719053:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787829071.719058:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829071.719059:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d13ec3c0 x1874670646793216/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:0.0:1787829071.719063:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1787829071.719070:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829071.719081:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787829071.719083:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829071.719086:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829071.719090:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00080000:1.0:1787829071.719090:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1787829071.719094:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839541348672 : -126234168202944 : ffff8d30d13ee940) 00000100:00080000:1.0:1787829071.719097:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1787829071.719100:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:1.0:1787829071.719101:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1787829071.719105:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787829071.719107:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1787829071.719110:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3872 00000100:00080000:1.0:1787829071.719129:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1787829071.719131:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787829071.719132:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:1.0:1787829071.719134:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:1.0:1787829071.719135:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:1.0:1787829071.719139:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000800:00000100:0.0:1787829071.719140:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000020:00000001:1.0:1787829071.719145:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:1.0:1787829071.719148:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787829071.719150:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:0.0:1787829071.719156:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00080000:1.0:1787829071.719156:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:1.0:1787829071.719162:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829071.719163:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 02000000:00000001:1.0:1787829071.719164:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:0.0:1787829071.719169:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829071.719170:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787829071.719174:0:2390:0:(events.c:68:request_out_callback()) Process leaving 02000000:00000001:1.0:1787829071.719176:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829071.719179:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1787829071.719183:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000008:00100000:1.0:1787829071.719183:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1787829071.719191:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829071.719192:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4298545920 00000100:00000001:1.0:1787829071.719193:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829071.719202:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1787829071.719203:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829071.719205:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 00000100:00000001:1.0:1787829071.719206:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1787829071.719210:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1787829071.719214:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829071.719216:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4298545920 00000100:00000001:1.0:1787829071.719219:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829071.719223:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1787829071.719229:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787829071.719232:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829071.719234:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787829071.719237:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d13ec780 x1874670646793344/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829071.719242:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829071/real 1787829071] req@ffff8d30d13ee940 x1874670646793088/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829096 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787829071.719267:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000400:00000001:0.0:1787829071.719271:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000001:1.0:1787829071.719272:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829071.719276:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1787829071.719279:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829071.719281:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1787829071.719282:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000400:00000001:0.0:1787829071.719284:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00080000:1.0:1787829071.719285:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787829071.719290:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:0.0:1787829071.719292:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00080000:1.0:1787829071.719296:0:468:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3872 00000100:00000001:0.0:1787829071.719298:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787829071.719304:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:1.0:1787829071.719304:0:468:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787829071.719307:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829071.719309:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829071.719311:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829071.719313:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:1.0:1787829071.719317:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:1.0:1787829071.719321:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00100000:0.0:1787829071.719323:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d13ee940 x1874670646793088/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829096 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:1.0:1787829071.719323:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000020:00000001:1.0:1787829071.719328:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:1.0:1787829071.719334:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787829071.719337:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:1.0:1787829071.719342:0:468:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:1.0:1787829071.719344:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829071.719345:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1787829071.719347:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829071.719353:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829071.719358:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:1.0:1787829071.719362:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00000001:0.0:1787829071.719365:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000008:00000001:1.0:1787829071.719366:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829071.719371:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829071.719373:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 02000000:00000001:1.0:1787829071.719375:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1787829071.719378:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829071.719383:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829071.719384:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1787829071.719386:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:0.0:1787829071.719391:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 02000000:00000001:1.0:1787829071.719391:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829071.719395:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829071.719398:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829071.719401:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00100000:1.0:1787829071.719407:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d13ec000 x1874670646793472/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787829071.719408:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787829071.719419:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00000001:1.0:1787829071.719421:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787829071.719427:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829071.719430:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787829071.719432:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787829071.719439:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787829071.719445:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d13ee940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646793088:192.168.203.143@tcp:250:kworker.0 00000100:00000001:0.0:1787829071.719454:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829071.719456:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787829071.719458:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787829071.719461:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787829071.719465:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787829071.719467:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787829071.719470:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787829071.719474:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787829071.719476:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787829071.719479:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787829071.719480:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829071.719482:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829071.719507:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787829071.719508:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829071.719511:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787829071.719512:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787829071.719514:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1787829071.719515:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787829071.719517:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829071.719519:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787829071.719521:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787829071.719523:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787829071.719526:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d13ec3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646793216:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787829071.719530:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787829071.719532:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787829071.719534:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829071.719535:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1787829071.719538:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829071.719545:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787829071.719546:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839541339072 : -126234168212544 : ffff8d30d13ec3c0) 00000100:00000001:0.0:1787829071.719550:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1787829071.719565:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:0.0:1787829071.719569:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1787829071.719572:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787829071.719573:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829071.719575:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787829071.719578:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829071.719582:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298544662 00000100:00000001:0.0:1787829071.719584:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298545920 00000100:00000001:0.0:1787829071.719587:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298544662 00000100:00000001:0.0:1787829071.719589:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298545920 00000100:00000001:0.0:1787829071.719591:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787829071.719593:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829071.719595:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787829071.719661:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829071/real 1787829071] req@ffff8d30d13ec3c0 x1874670646793216/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829096 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787829071.719671:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787829071.719672:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787829071.719675:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787829071.719677:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787829071.719678:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829071.719680:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787829071.719681:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787829071.719685:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d13ec3c0 x1874670646793216/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829096 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787829071.719692:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829071.719694:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787829071.719696:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829071.719698:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787829071.719699:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829071.719700:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787829071.719702:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787829071.719704:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829071.719706:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787829071.719708:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787829071.719727:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d13ec3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646793216:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787829071.719731:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1787829071.719732:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787829071.719734:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829071.719736:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787829071.719738:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787829071.719741:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787829071.719747:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d13ec780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646793344:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787829071.719751:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787829071.719754:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787829071.719756:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829071.719759:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1787829071.719763:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829071.719770:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787829071.719772:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839541340032 : -126234168211584 : ffff8d30d13ec780) 00000100:00000001:0.0:1787829071.719793:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1787829071.719813:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:0.0:1787829071.719820:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1787829071.719824:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787829071.719826:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829071.719829:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787829071.719833:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829071.719839:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298544663 00000100:00000001:0.0:1787829071.719841:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298545920 00000100:00000001:0.0:1787829071.719843:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298544663 00000100:00000001:0.0:1787829071.719846:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298545920 00000100:00000001:0.0:1787829071.719848:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787829071.719850:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829071.719851:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787829071.719856:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829071/real 1787829071] req@ffff8d30d13ec780 x1874670646793344/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829096 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787829071.719864:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787829071.719866:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787829071.719868:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787829071.719870:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787829071.719871:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829071.719873:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787829071.719874:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787829071.719878:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d13ec780 x1874670646793344/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829096 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787829071.719892:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829071.719894:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787829071.719896:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829071.719898:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787829071.719899:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829071.719900:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787829071.719902:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787829071.719904:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829071.719907:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787829071.719909:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787829071.719912:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d13ec780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646793344:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787829071.719915:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1787829071.719916:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787829071.719918:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829071.719920:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787829071.719922:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787829071.719924:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787829071.719927:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d13ec000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646793472:192.168.203.143@tcp:8:kworker.0 00000100:00000001:0.0:1787829071.719929:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787829071.719931:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787829071.719933:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829071.719935:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1787829071.719938:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829071.719943:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787829071.719944:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839541338112 : -126234168213504 : ffff8d30d13ec000) 00000100:00000001:0.0:1787829071.719947:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1787829071.719976:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:0.0:1787829071.719980:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1787829071.719981:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787829071.719983:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829071.719985:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787829071.719987:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829071.719990:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298544676 00000100:00000001:0.0:1787829071.719992:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298545920 00000100:00000001:0.0:1787829071.719994:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298544676 00000100:00000001:0.0:1787829071.719996:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298545920 00000100:00000001:0.0:1787829071.719998:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787829071.720000:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829071.720002:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787829071.720005:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829071/real 1787829071] req@ffff8d30d13ec000 x1874670646793472/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829096 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787829071.720011:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787829071.720012:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787829071.720014:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787829071.720031:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787829071.720033:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829071.720034:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787829071.720035:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787829071.720039:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d13ec000 x1874670646793472/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829096 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787829071.720044:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829071.720046:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787829071.720048:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829071.720049:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787829071.720050:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829071.720051:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787829071.720053:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787829071.720055:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829071.720057:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787829071.720058:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787829071.720062:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d13ec000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646793472:192.168.203.143@tcp:8:kworker.0 00000100:00000001:0.0:1787829071.720065:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829071.720067:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787829071.720068:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787829071.720070:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787829071.720071:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787829071.720072:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787829071.720073:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787829071.720075:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787829071.720076:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787829071.720078:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787829071.720079:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829071.720081:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787829071.720082:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787829071.720083:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787829071.720084:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787829071.720085:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787829071.720086:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787829071.720088:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787829071.720089:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787829071.720090:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787829071.720091:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829071.720092:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787829071.720093:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787829071.720094:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787829071.720095:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787829071.720096:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787829071.720097:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787829071.720098:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787829071.720099:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787829071.720100:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787829071.720101:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829071.720102:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829071.720106:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787829071.720107:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829071.720108:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787829071.720110:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829072.614701:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829072.614706:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829072.614750:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829072.614753:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829072.614755:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829073.638782:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829073.638793:0:815:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1787829073.638797:0:815:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1787829073.638801:0:815:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829073.638807:0:815:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1787829073.638811:0:815:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829073.638814:0:815:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829073.638817:0:815:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829073.638901:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829073.638906:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829073.638911:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829074.662710:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829074.662715:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829074.662765:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829074.662768:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829074.662770:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829075.686676:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829075.686682:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829075.686728:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829075.686731:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829075.686734:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829076.710696:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829076.710701:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829076.710756:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829076.710759:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829076.710762:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787829076.838752:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829076.838760:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1787829076.838762:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787829076.838765:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787829076.838769:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787829076.838775:0:468:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 3877 00000100:00080000:3.0:1787829076.838783:0:468:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787829076.838785:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829076.838787:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:3.0:1787829076.838791:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1787829076.838793:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1787829076.838796:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:3.0:1787829076.838798:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:3.0:1787829076.838799:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829076.838801:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:3.0:1787829076.838805:0:468:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:3.0:1787829076.838806:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829076.838808:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787829076.838811:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829076.838814:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829076.838815:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:3.0:1787829076.838817:0:468:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829076.838824:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787829076.838825:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829076.838827:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787829076.838830:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787829076.838834:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829076.838843:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829076.838850:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c7015680 x1874670646793600/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787829076.838859:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787829076.838862:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829076.838876:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787829076.838879:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829076.838882:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1787829076.838884:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787829076.838885:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787829076.838887:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787829076.838889:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3877 00000100:00080000:3.0:1787829076.838892:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787829076.838893:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829076.838894:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:3.0:1787829076.838896:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1787829076.838897:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1787829076.838899:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:1.0:1787829076.838900:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:3.0:1787829076.838901:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:3.0:1787829076.838904:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787829076.838906:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787829076.838906:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:1.0:1787829076.838911:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00080000:3.0:1787829076.838911:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:1.0:1787829076.838915:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787829076.838916:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829076.838918:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829076.838919:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1787829076.838924:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787829076.838925:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829076.838927:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829076.838929:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829076.838932:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829076.838938:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c7015680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646793600:192.168.203.143@tcp:250:kworker.0 00000100:00000001:3.0:1787829076.838939:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:1.0:1787829076.838944:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000008:00100000:3.0:1787829076.838946:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 02000000:00000001:1.0:1787829076.838948:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000008:00000001:3.0:1787829076.838950:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829076.838952:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829076.838953:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829076.838955:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1787829076.838958:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787829076.838960:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829076.838961:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829076.838962:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787829076.838965:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787829076.838966:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829076.838971:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829076.838975:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c70143c0 x1874670646793728/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787829076.838977:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787829076.838982:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369541248 : -126234340010368 : ffff8d30c7015680) 00000100:00000001:1.0:1787829076.838989:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:3.0:1787829076.838992:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787829076.838995:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829076.838997:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787829076.838999:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829076.839001:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1787829076.839002:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787829076.839003:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787829076.839005:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787829076.839007:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3877 00000100:00080000:3.0:1787829076.839010:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787829076.839011:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829076.839012:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:3.0:1787829076.839014:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1787829076.839015:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1787829076.839017:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000020:00000001:3.0:1787829076.839019:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:3.0:1787829076.839020:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829076.839021:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000800:00000100:1.0:1787829076.839031:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787829076.839039:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787829076.839043:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829076.839045:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829076.839046:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787829076.839050:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829076.839054:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4298551040 00000100:00000001:1.0:1787829076.839061:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 00000100:00000001:1.0:1787829076.839065:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4298551040 00000100:00000001:1.0:1787829076.839068:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 00000100:00000001:1.0:1787829076.839071:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829076.839073:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829076.839075:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787829076.839079:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829076/real 1787829076] req@ffff8d30c7015680 x1874670646793600/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829101 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787829076.839089:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00080000:3.0:1787829076.839089:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000400:00000001:1.0:1787829076.839093:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829076.839093:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829076.839097:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1787829076.839100:0:2390:0:(events.c:78:reply_in_callback()) Process entered 02000000:00000001:3.0:1787829076.839100:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829076.839104:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829076.839106:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829076.839106:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:3.0:1787829076.839108:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00000001:1.0:1787829076.839110:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000008:00000001:3.0:1787829076.839111:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829076.839116:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829076.839118:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 02000000:00000001:3.0:1787829076.839121:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:1.0:1787829076.839122:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:3.0:1787829076.839123:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829076.839127:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787829076.839128:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787829076.839130:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829076.839133:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c7015680 x1874670646793600/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829101 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829076.839134:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829076.839142:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c7014f00 x1874670646793856/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787829076.839156:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829076.839158:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787829076.839160:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829076.839161:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829076.839166:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829076.839166:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787829076.839169:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1787829076.839170:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829076.839173:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:3.0:1787829076.839174:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1787829076.839176:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829076.839177:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:1.0:1787829076.839179:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829076.839180:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829076.839186:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00000001:3.0:1787829076.839186:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787829076.839190:0:468:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3877 00000100:00080000:1.0:1787829076.839194:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00080000:3.0:1787829076.839194:0:468:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1787829076.839198:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000001:3.0:1787829076.839198:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829076.839201:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00100000:1.0:1787829076.839204:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c7015680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646793600:192.168.203.143@tcp:250:kworker.0 00000020:00000001:3.0:1787829076.839205:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1787829076.839208:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1787829076.839211:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:1.0:1787829076.839213:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:3.0:1787829076.839214:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:1.0:1787829076.839217:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829076.839217:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787829076.839220:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 00000100:00000001:3.0:1787829076.839220:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 02000000:00000001:1.0:1787829076.839224:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 00000100:00080000:3.0:1787829076.839226:0:468:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 02000000:00000001:1.0:1787829076.839228:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00000001:3.0:1787829076.839230:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829076.839232:0:2390:0:(genops.c:1217:class_import_put()) Process entered 02000000:00000001:3.0:1787829076.839232:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000020:00000001:1.0:1787829076.839235:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829076.839236:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829076.839239:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829076.839240:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787829076.839243:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829076.839244:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:1.0:1787829076.839246:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000008:00100000:3.0:1787829076.839247:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00000001:1.0:1787829076.839250:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000008:00000001:3.0:1787829076.839251:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829076.839254:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829076.839254:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829076.839261:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:1.0:1787829076.839263:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787829076.839266:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829076.839267:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787829076.839269:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1787829076.839270:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787829076.839271:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787829076.839272:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829076.839273:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787829076.839275:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829076.839276:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829076.839279:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c70143c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646793728:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787829076.839281:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787829076.839283:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787829076.839284:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829076.839285:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787829076.839288:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829076.839293:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787829076.839294:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369536448 : -126234340015168 : ffff8d30c70143c0) 00000100:00000001:1.0:1787829076.839298:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1787829076.839309:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787829076.839312:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787829076.839314:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829076.839315:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829076.839317:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787829076.839318:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829076.839322:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298549782 00000100:00000001:1.0:1787829076.839324:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298551040 00000100:00000001:1.0:1787829076.839326:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298549782 00000100:00000001:1.0:1787829076.839327:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298551040 00000100:00000001:1.0:1787829076.839329:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829076.839330:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829076.839331:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787829076.839335:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829076/real 1787829076] req@ffff8d30c70143c0 x1874670646793728/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829101 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787829076.839340:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787829076.839341:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787829076.839344:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787829076.839345:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787829076.839346:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829076.839348:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829076.839349:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787829076.839351:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c70143c0 x1874670646793728/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829101 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1787829076.839355:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829076.839357:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829076.839359:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:1.0:1787829076.839360:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 02000000:00000001:3.0:1787829076.839363:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:1.0:1787829076.839364:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829076.839368:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 02000000:00000001:3.0:1787829076.839368:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829076.839370:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829076.839374:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787829076.839375:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787829076.839377:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00000001:3.0:1787829076.839378:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1787829076.839381:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787829076.839385:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787829076.839391:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c70143c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646793728:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787829076.839394:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787829076.839395:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787829076.839396:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829076.839397:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787829076.839399:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829076.839400:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829076.839402:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c7014f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646793856:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787829076.839405:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787829076.839407:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787829076.839408:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829076.839409:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787829076.839412:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829076.839416:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787829076.839417:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369539328 : -126234340012288 : ffff8d30c7014f00) 00000100:00000001:1.0:1787829076.839420:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00100000:3.0:1787829076.839427:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c7015a40 x1874670646793984/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787829076.839433:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787829076.839441:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787829076.839441:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787829076.839444:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829076.839447:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829076.839450:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829076.839451:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829076.839456:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787829076.839459:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829076.839461:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298549782 00000100:00000001:1.0:1787829076.839463:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298551040 00000100:00000001:1.0:1787829076.839465:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298549782 00000100:00000001:1.0:1787829076.839466:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298551040 00000100:00000001:1.0:1787829076.839469:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829076.839471:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829076.839472:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787829076.839474:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829076/real 1787829076] req@ffff8d30c7014f00 x1874670646793856/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829101 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787829076.839485:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787829076.839487:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787829076.839490:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787829076.839492:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787829076.839494:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829076.839495:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829076.839496:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787829076.839499:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c7014f00 x1874670646793856/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829101 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787829076.839504:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829076.839506:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787829076.839508:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829076.839509:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829076.839511:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829076.839512:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787829076.839512:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787829076.839514:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829076.839516:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787829076.839519:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787829076.839521:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c7014f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646793856:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787829076.839524:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829076.839525:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829076.839526:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787829076.839527:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787829076.839529:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787829076.839530:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787829076.839531:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787829076.839532:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787829076.839533:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787829076.839534:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787829076.839535:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829076.839537:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829076.839538:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787829076.839540:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787829076.839541:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787829076.839543:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787829076.839544:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787829076.839546:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787829076.839547:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787829076.839549:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787829076.839550:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829076.839551:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829076.839554:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787829076.839555:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829076.839557:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787829076.839558:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1787829076.839559:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787829076.839561:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787829076.839562:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829076.839581:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787829076.839583:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829076.839586:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829076.839590:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c7015a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646793984:192.168.203.143@tcp:8:kworker.0 00000100:00000001:1.0:1787829076.839592:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787829076.839594:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787829076.839595:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829076.839596:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787829076.839598:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829076.839623:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787829076.839624:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369542208 : -126234340009408 : ffff8d30c7015a40) 00000100:00000001:1.0:1787829076.839627:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1787829076.839638:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787829076.839643:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787829076.839646:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829076.839647:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829076.839649:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787829076.839651:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829076.839653:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298549796 00000100:00000001:1.0:1787829076.839655:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298551040 00000100:00000001:1.0:1787829076.839657:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298549796 00000100:00000001:1.0:1787829076.839658:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298551040 00000100:00000001:1.0:1787829076.839660:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829076.839661:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829076.839663:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787829076.839665:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829076/real 1787829076] req@ffff8d30c7015a40 x1874670646793984/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829101 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787829076.839670:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787829076.839671:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787829076.839672:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787829076.839673:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787829076.839674:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829076.839675:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829076.839676:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787829076.839679:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c7015a40 x1874670646793984/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829101 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787829076.839683:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829076.839684:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787829076.839686:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829076.839687:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829076.839688:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829076.839689:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787829076.839690:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787829076.839692:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829076.839693:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787829076.839694:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787829076.839696:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c7015a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646793984:192.168.203.143@tcp:8:kworker.0 00000100:00000001:1.0:1787829076.839698:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829076.839700:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829076.839701:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787829076.839702:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787829076.839703:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787829076.839704:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787829076.839705:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787829076.839706:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787829076.839707:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787829076.839708:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787829076.839709:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829076.839710:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829076.839713:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787829076.839714:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829076.839715:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787829076.839716:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829077.734690:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829077.734696:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829077.734760:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829077.734764:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829077.734768:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829078.758686:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829078.758690:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829078.758745:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829078.758748:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829078.758751:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829079.782686:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829079.782691:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829079.782756:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829079.782760:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829079.782762:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829080.806723:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829080.806730:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829080.806793:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829080.806797:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829080.806800:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829081.830723:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829081.830729:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829081.830809:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829081.830813:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829081.830816:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787829081.958681:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829081.958689:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1787829081.958692:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787829081.958695:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787829081.958699:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787829081.958704:0:468:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 3882 00000100:00080000:3.0:1787829081.958713:0:468:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787829081.958715:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829081.958716:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:3.0:1787829081.958720:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1787829081.958722:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1787829081.958725:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:3.0:1787829081.958727:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:3.0:1787829081.958728:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829081.958729:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:3.0:1787829081.958733:0:468:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:3.0:1787829081.958734:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829081.958736:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787829081.958738:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829081.958741:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829081.958742:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:3.0:1787829081.958744:0:468:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829081.958754:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787829081.958756:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829081.958758:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787829081.958760:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787829081.958763:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829081.958772:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829081.958780:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c70161c0 x1874670646794112/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787829081.958787:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787829081.958791:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829081.958815:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787829081.958817:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829081.958820:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1787829081.958821:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787829081.958822:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787829081.958824:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787829081.958826:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3882 00000100:00080000:3.0:1787829081.958829:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787829081.958831:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829081.958832:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:3.0:1787829081.958834:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1787829081.958836:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:1.0:1787829081.958838:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:3.0:1787829081.958838:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000020:00000001:3.0:1787829081.958843:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:1.0:1787829081.958845:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787829081.958846:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787829081.958849:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787829081.958849:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:3.0:1787829081.958853:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:1.0:1787829081.958854:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787829081.958857:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829081.958858:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829081.958861:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1787829081.958864:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787829081.958866:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829081.958867:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829081.958871:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829081.958872:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829081.958874:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00100000:1.0:1787829081.958879:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c70161c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646794112:192.168.203.143@tcp:250:kworker.0 00000008:00100000:3.0:1787829081.958879:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1787829081.958885:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829081.958888:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:3.0:1787829081.958888:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829081.958894:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1787829081.958895:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1787829081.958895:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829081.958899:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1787829081.958900:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829081.958902:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1787829081.958904:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1787829081.958905:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829081.958909:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829081.958911:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829081.958918:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c7016580 x1874670646794240/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787829081.958924:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787829081.958924:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787829081.958926:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829081.958927:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369544128 : -126234340007488 : ffff8d30c70161c0) 00000100:00000001:3.0:1787829081.958930:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829081.958942:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00080000:3.0:1787829081.958943:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829081.958947:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1787829081.958948:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787829081.958950:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787829081.958952:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787829081.958953:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3882 00000100:00080000:3.0:1787829081.958956:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787829081.958958:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829081.958959:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:3.0:1787829081.958960:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1787829081.958961:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1787829081.958963:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000020:00000001:3.0:1787829081.958964:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:3.0:1787829081.958966:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829081.958967:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:3.0:1787829081.958970:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:3.0:1787829081.958972:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829081.958974:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000800:00000100:1.0:1787829081.958976:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 02000000:00000001:3.0:1787829081.958987:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829081.958988:0:2390:0:(events.c:34:request_out_callback()) Process entered 02000000:00000001:3.0:1787829081.958992:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829081.958993:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829081.959006:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:1.0:1787829081.959008:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829081.959011:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000008:00100000:3.0:1787829081.959012:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00000001:1.0:1787829081.959015:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:3.0:1787829081.959017:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829081.959020:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4298556160 00000100:00000001:3.0:1787829081.959021:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829081.959030:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:1.0:1787829081.959031:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 02000000:00000001:3.0:1787829081.959032:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829081.959037:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4298556160 00000100:00000001:3.0:1787829081.959037:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787829081.959039:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:1.0:1787829081.959044:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 02000000:00000001:3.0:1787829081.959045:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829081.959049:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829081.959052:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829081.959053:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829081.959054:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787829081.959060:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829081/real 1787829081] req@ffff8d30c70161c0 x1874670646794112/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829106 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787829081.959063:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c7015a40 x1874670646794368/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787829081.959098:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000400:00000001:1.0:1787829081.959101:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000001:3.0:1787829081.959101:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:1.0:1787829081.959104:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829081.959106:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829081.959109:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00080000:3.0:1787829081.959110:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1787829081.959112:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00080000:3.0:1787829081.959113:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1787829081.959115:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829081.959116:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:1.0:1787829081.959120:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00080000:3.0:1787829081.959120:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787829081.959124:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829081.959125:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787829081.959131:0:468:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3882 00000100:00100000:1.0:1787829081.959132:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c70161c0 x1874670646794112/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829106 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787829081.959135:0:468:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787829081.959140:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829081.959142:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:1.0:1787829081.959145:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:3.0:1787829081.959146:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1787829081.959149:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:1.0:1787829081.959151:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000020:00000001:3.0:1787829081.959152:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000020:00000001:3.0:1787829081.959156:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:3.0:1787829081.959157:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829081.959158:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:3.0:1787829081.959160:0:468:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:3.0:1787829081.959161:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829081.959163:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1787829081.959165:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:3.0:1787829081.959167:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829081.959168:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829081.959171:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:3.0:1787829081.959172:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829081.959175:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829081.959176:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:1.0:1787829081.959178:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000008:00100000:3.0:1787829081.959179:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1787829081.959185:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1787829081.959186:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00000001:3.0:1787829081.959187:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1787829081.959201:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 02000000:00000001:3.0:1787829081.959201:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787829081.959204:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829081.959206:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000001:3.0:1787829081.959207:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787829081.959210:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00100000:1.0:1787829081.959215:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c70161c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646794112:192.168.203.143@tcp:250:kworker.0 02000000:00000001:3.0:1787829081.959215:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829081.959221:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829081.959223:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829081.959225:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829081.959227:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 00000100:00100000:3.0:1787829081.959229:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c7014f00 x1874670646794496/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:1.0:1787829081.959231:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787829081.959235:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787829081.959238:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000100:00000001:3.0:1787829081.959239:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000020:00000001:1.0:1787829081.959240:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 00000100:00000001:3.0:1787829081.959244:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829081.959246:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 00000100:00000001:3.0:1787829081.959247:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829081.959251:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787829081.959254:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787829081.959256:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829081.959257:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829081.959275:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787829081.959277:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829081.959279:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787829081.959280:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1787829081.959281:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787829081.959282:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787829081.959283:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829081.959285:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787829081.959286:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829081.959288:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829081.959291:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c7016580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646794240:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787829081.959293:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787829081.959295:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787829081.959296:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829081.959297:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787829081.959299:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829081.959304:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787829081.959305:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369545088 : -126234340006528 : ffff8d30c7016580) 00000100:00000001:1.0:1787829081.959308:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1787829081.959316:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787829081.959319:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787829081.959321:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829081.959323:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829081.959324:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787829081.959326:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829081.959329:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298554902 00000100:00000001:1.0:1787829081.959331:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298556160 00000100:00000001:1.0:1787829081.959333:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298554902 00000100:00000001:1.0:1787829081.959334:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298556160 00000100:00000001:1.0:1787829081.959336:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829081.959337:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829081.959338:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787829081.959341:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829081/real 1787829081] req@ffff8d30c7016580 x1874670646794240/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829106 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787829081.959346:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787829081.959347:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787829081.959349:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787829081.959350:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787829081.959351:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829081.959352:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829081.959353:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787829081.959355:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c7016580 x1874670646794240/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829106 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787829081.959360:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829081.959362:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787829081.959363:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829081.959364:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829081.959365:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829081.959366:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787829081.959367:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787829081.959369:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829081.959371:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787829081.959372:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787829081.959375:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c7016580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646794240:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787829081.959377:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787829081.959378:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787829081.959379:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829081.959380:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787829081.959381:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829081.959383:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829081.959392:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c7015a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646794368:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787829081.959394:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787829081.959395:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787829081.959396:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829081.959397:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787829081.959398:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829081.959401:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787829081.959402:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369542208 : -126234340009408 : ffff8d30c7015a40) 00000100:00000001:1.0:1787829081.959404:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1787829081.959409:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787829081.959412:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787829081.959413:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829081.959414:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829081.959415:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787829081.959417:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829081.959419:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298554902 00000100:00000001:1.0:1787829081.959421:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298556160 00000100:00000001:1.0:1787829081.959422:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298554902 00000100:00000001:1.0:1787829081.959424:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298556160 00000100:00000001:1.0:1787829081.959425:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829081.959427:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829081.959428:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787829081.959430:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829081/real 1787829081] req@ffff8d30c7015a40 x1874670646794368/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829106 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787829081.959465:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787829081.959466:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787829081.959468:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787829081.959469:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787829081.959471:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829081.959472:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829081.959473:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787829081.959476:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c7015a40 x1874670646794368/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829106 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787829081.959482:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829081.959484:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787829081.959485:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829081.959487:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829081.959488:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829081.959489:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787829081.959491:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787829081.959493:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829081.959495:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787829081.959497:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787829081.959499:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c7015a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646794368:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787829081.959502:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787829081.959503:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787829081.959504:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829081.959506:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787829081.959507:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829081.959509:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829081.959512:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c7014f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646794496:192.168.203.143@tcp:8:kworker.0 00000100:00000001:1.0:1787829081.959514:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787829081.959516:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787829081.959518:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829081.959519:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787829081.959522:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829081.959526:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787829081.959558:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369539328 : -126234340012288 : ffff8d30c7014f00) 00000100:00000001:1.0:1787829081.959561:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1787829081.959570:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787829081.959573:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787829081.959574:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829081.959575:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829081.959577:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787829081.959578:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829081.959591:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298554915 00000100:00000001:1.0:1787829081.959593:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298556160 00000100:00000001:1.0:1787829081.959594:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298554915 00000100:00000001:1.0:1787829081.959596:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298556160 00000100:00000001:1.0:1787829081.959597:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829081.959599:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829081.959617:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787829081.959620:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829081/real 1787829081] req@ffff8d30c7014f00 x1874670646794496/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829106 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787829081.959626:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787829081.959627:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787829081.959628:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787829081.959629:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787829081.959631:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829081.959632:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829081.959633:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787829081.959635:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c7014f00 x1874670646794496/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829106 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787829081.959640:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829081.959641:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787829081.959643:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829081.959644:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829081.959645:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829081.959646:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787829081.959647:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787829081.959649:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829081.959651:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787829081.959652:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787829081.959654:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c7014f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646794496:192.168.203.143@tcp:8:kworker.0 00000100:00000001:1.0:1787829081.959656:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829081.959658:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829081.959658:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787829081.959659:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787829081.959661:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787829081.959662:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787829081.959663:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787829081.959664:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787829081.959665:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787829081.959667:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787829081.959668:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829081.959669:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829081.959670:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787829081.959670:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787829081.959672:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787829081.959673:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787829081.959674:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787829081.959674:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787829081.959675:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787829081.959676:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787829081.959677:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829081.959678:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829081.959679:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787829081.959679:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787829081.959681:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787829081.959682:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787829081.959682:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787829081.959683:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787829081.959684:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787829081.959686:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787829081.959687:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829081.959688:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829081.959690:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787829081.959691:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829081.959693:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787829081.959694:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829082.854711:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829082.854716:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829082.854802:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829082.854806:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829082.854810:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829083.878734:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829083.878740:0:815:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1787829083.878742:0:815:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1787829083.878744:0:815:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829083.878748:0:815:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1787829083.878750:0:815:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829083.878751:0:815:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829083.878753:0:815:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829083.878818:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829083.878822:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829083.878825:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829084.902694:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829084.902701:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829084.902753:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829084.902756:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829084.902759:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829085.926668:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829085.926674:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829085.926722:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829085.926726:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829085.926729:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829086.950714:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829086.950720:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829086.950774:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829086.950778:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829086.950780:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787829087.078680:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829087.078688:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1787829087.078691:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787829087.078694:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787829087.078697:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787829087.078703:0:40798:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 3887 00000100:00080000:0.0:1787829087.078711:0:40798:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787829087.078714:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829087.078715:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787829087.078719:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787829087.078721:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787829087.078723:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:0.0:1787829087.078724:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:0.0:1787829087.078726:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829087.078726:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:0.0:1787829087.078730:0:40798:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787829087.078732:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829087.078734:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787829087.078736:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787829087.078738:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829087.078740:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1787829087.078742:0:40798:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829087.078750:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787829087.078751:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829087.078753:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787829087.078756:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787829087.078759:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829087.078768:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787829087.078775:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d24f00 x1874670646794624/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787829087.078783:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787829087.078786:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829087.078797:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787829087.078799:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829087.078803:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1787829087.078804:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787829087.078805:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787829087.078807:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787829087.078809:0:40798:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3887 00000100:00080000:0.0:1787829087.078811:0:40798:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787829087.078812:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829087.078813:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787829087.078814:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787829087.078815:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787829087.078816:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000020:00000001:0.0:1787829087.078818:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:0.0:1787829087.078819:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829087.078820:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:0.0:1787829087.078822:0:40798:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787829087.078823:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829087.078824:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787829087.078828:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787829087.078830:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829087.078831:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1787829087.078834:0:40798:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1787829087.078836:0:40798:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829087.078837:0:40798:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829087.078839:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787829087.078840:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829087.078841:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787829087.078842:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787829087.078844:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829087.078847:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787829087.078850:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d25a40 x1874670646794752/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787829087.078860:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787829087.078862:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829087.078863:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787829087.078864:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787829087.078868:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1787829087.078869:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00080000:0.0:1787829087.078871:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:1.0:1787829087.078872:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1787829087.078873:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:1.0:1787829087.078876:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00080000:0.0:1787829087.078877:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787829087.078879:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829087.078880:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787829087.078884:0:40798:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3887 02000000:00000001:1.0:1787829087.078884:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000100:00080000:0.0:1787829087.078887:0:40798:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 02000000:00000001:1.0:1787829087.078888:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787829087.078890:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 02000000:00000001:1.0:1787829087.078892:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829087.078893:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787829087.078896:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787829087.078898:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00100000:1.0:1787829087.078899:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d24f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646794624:192.168.203.143@tcp:250:kworker.0 00000020:00000001:0.0:1787829087.078900:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000020:00000001:0.0:1787829087.078904:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:1.0:1787829087.078907:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:0.0:1787829087.078908:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829087.078911:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 02000000:00000001:1.0:1787829087.078912:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000100:00080000:0.0:1787829087.078915:0:40798:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 02000000:00000001:1.0:1787829087.078916:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829087.078919:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829087.078920:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1787829087.078922:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787829087.078926:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829087.078926:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829087.078932:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829087.078933:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1787829087.078934:0:40798:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1787829087.078936:0:40798:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829087.078937:0:40798:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829087.078940:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787829087.078941:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829087.078943:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787829087.078944:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:1.0:1787829087.078947:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601340160 : -126234108211456 : ffff8d30d4d24f00) 02000000:00000001:0.0:1787829087.078948:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787829087.078951:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829087.078953:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:0.0:1787829087.078955:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787829087.078960:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d270c0 x1874670646794880/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787829087.078964:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787829087.078966:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829087.078968:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787829087.078970:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829087.078972:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1787829087.078973:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787829087.078974:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787829087.078977:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787829087.078978:0:40798:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3887 00000100:00080000:0.0:1787829087.078981:0:40798:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787829087.078982:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829087.078984:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000800:00000100:1.0:1787829087.078987:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000020:00000001:0.0:1787829087.078988:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787829087.078991:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787829087.078993:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000020:00000001:0.0:1787829087.078995:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:0.0:1787829087.078997:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829087.078998:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:1.0:1787829087.079000:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00080000:0.0:1787829087.079002:0:40798:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:1.0:1787829087.079005:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787829087.079006:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829087.079009:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1787829087.079010:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829087.079012:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787829087.079017:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829087.079019:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829087.079020:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4298561280 02000000:00000001:0.0:1787829087.079023:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829087.079025:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1787829087.079026:0:40798:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1787829087.079029:0:40798:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829087.079030:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1787829087.079031:0:40798:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829087.079037:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4298561280 02000000:00000001:0.0:1787829087.079039:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787829087.079041:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829087.079043:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:1.0:1787829087.079043:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 02000000:00000001:0.0:1787829087.079045:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787829087.079049:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829087.079049:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829087.079055:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829087.079057:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829087.079057:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787829087.079063:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829087/real 1787829087] req@ffff8d30d4d24f00 x1874670646794624/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829112 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829087.079064:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d24000 x1874670646795008/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787829087.079091:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787829087.079094:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:1.0:1787829087.079095:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000001:0.0:1787829087.079097:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:1.0:1787829087.079098:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787829087.079103:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787829087.079105:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787829087.079107:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829087.079109:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829087.079110:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787829087.079114:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d24f00 x1874670646794624/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829112 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787829087.079121:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829087.079123:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787829087.079125:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829087.079126:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829087.079127:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829087.079129:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787829087.079130:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787829087.079132:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829087.079134:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787829087.079136:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787829087.079138:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d24f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646794624:192.168.203.143@tcp:250:kworker.0 00000100:00000001:1.0:1787829087.079142:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787829087.079143:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787829087.079144:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829087.079145:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787829087.079146:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829087.079148:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829087.079150:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d25a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646794752:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787829087.079152:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787829087.079153:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787829087.079155:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829087.079156:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787829087.079158:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829087.079161:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787829087.079162:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601343040 : -126234108208576 : ffff8d30d4d25a40) 00000100:00000001:1.0:1787829087.079165:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1787829087.079172:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787829087.079175:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787829087.079176:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829087.079177:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829087.079178:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787829087.079180:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829087.079182:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298560022 00000100:00000001:1.0:1787829087.079184:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298561280 00000100:00000001:1.0:1787829087.079186:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298560022 00000100:00000001:1.0:1787829087.079187:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298561280 00000100:00000001:1.0:1787829087.079189:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829087.079190:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829087.079191:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787829087.079193:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829087/real 1787829087] req@ffff8d30d4d25a40 x1874670646794752/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829112 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787829087.079198:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787829087.079199:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787829087.079200:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787829087.079201:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787829087.079202:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829087.079203:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829087.079204:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787829087.079207:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d25a40 x1874670646794752/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829112 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787829087.079212:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829087.079213:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787829087.079215:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829087.079216:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829087.079216:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829087.079217:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787829087.079218:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787829087.079220:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829087.079222:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787829087.079223:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787829087.079225:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d25a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646794752:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787829087.079227:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829087.079229:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829087.079230:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787829087.079231:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787829087.079233:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787829087.079235:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787829087.079236:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787829087.079239:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787829087.079240:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787829087.079243:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787829087.079244:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829087.079244:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829087.079245:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787829087.079246:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787829087.079247:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787829087.079248:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787829087.079248:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787829087.079249:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787829087.079251:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787829087.079252:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787829087.079253:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829087.079254:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829087.079260:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787829087.079261:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829087.079263:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787829087.079264:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1787829087.079265:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787829087.079266:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787829087.079267:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829087.079268:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787829087.079269:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829087.079270:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829087.079272:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d270c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646794880:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787829087.079274:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787829087.079275:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787829087.079276:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829087.079277:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787829087.079279:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829087.079282:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787829087.079283:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601348800 : -126234108202816 : ffff8d30d4d270c0) 00000100:00000001:1.0:1787829087.079285:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1787829087.079291:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787829087.079293:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787829087.079294:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829087.079295:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829087.079296:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787829087.079298:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829087.079300:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298560022 00000100:00000001:1.0:1787829087.079302:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298561280 00000100:00000001:1.0:1787829087.079303:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298560022 00000100:00000001:1.0:1787829087.079304:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298561280 00000100:00000001:1.0:1787829087.079306:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829087.079307:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829087.079308:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787829087.079310:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829087/real 1787829087] req@ffff8d30d4d270c0 x1874670646794880/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829112 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787829087.079319:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787829087.079320:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787829087.079321:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787829087.079322:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787829087.079323:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829087.079324:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829087.079325:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787829087.079327:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d270c0 x1874670646794880/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829112 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787829087.079331:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829087.079332:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787829087.079334:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829087.079335:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829087.079336:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829087.079337:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787829087.079338:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787829087.079340:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829087.079342:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787829087.079343:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787829087.079345:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d270c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646794880:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787829087.079347:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787829087.079347:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787829087.079348:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829087.079349:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787829087.079350:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829087.079351:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829087.079353:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d24000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646795008:192.168.203.143@tcp:8:kworker.0 00000100:00000001:1.0:1787829087.079355:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787829087.079356:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787829087.079357:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829087.079358:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787829087.079359:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829087.079362:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787829087.079363:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601336320 : -126234108215296 : ffff8d30d4d24000) 00000100:00000001:1.0:1787829087.079365:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1787829087.079370:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787829087.079372:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787829087.079374:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829087.079375:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829087.079376:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787829087.079377:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829087.079379:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298560035 00000100:00000001:1.0:1787829087.079380:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298561280 00000100:00000001:1.0:1787829087.079382:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298560035 00000100:00000001:1.0:1787829087.079383:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298561280 00000100:00000001:1.0:1787829087.079385:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829087.079386:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829087.079387:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787829087.079389:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829087/real 1787829087] req@ffff8d30d4d24000 x1874670646795008/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829112 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787829087.079394:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787829087.079394:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787829087.079395:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787829087.079396:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787829087.079397:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829087.079398:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829087.079399:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787829087.079401:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d24000 x1874670646795008/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829112 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787829087.079405:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829087.079406:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787829087.079408:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829087.079409:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829087.079410:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829087.079411:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787829087.079412:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787829087.079413:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829087.079415:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787829087.079416:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787829087.079418:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d24000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646795008:192.168.203.143@tcp:8:kworker.0 00000100:00000001:1.0:1787829087.079420:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829087.079421:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829087.079422:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787829087.079423:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787829087.079424:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787829087.079425:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787829087.079425:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787829087.079426:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787829087.079427:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787829087.079429:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787829087.079430:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829087.079431:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829087.079432:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787829087.079432:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787829087.079433:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787829087.079434:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787829087.079434:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787829087.079435:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787829087.079436:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787829087.079437:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787829087.079438:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829087.079439:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829087.079440:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787829087.079442:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829087.079443:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787829087.079444:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829087.974714:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829087.974722:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829087.974812:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829087.974815:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829087.974818:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829088.998716:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829088.998721:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829088.998783:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829088.998787:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829088.998790:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829090.022696:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829090.022702:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829090.022755:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829090.022760:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829090.022764:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829091.046714:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829091.046720:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829091.046850:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829091.046853:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829091.046856:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829092.070698:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829092.070704:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829092.070751:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829092.070756:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829092.070759:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787829092.198824:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829092.198839:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1787829092.198853:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787829092.198859:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787829092.198865:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787829092.198874:0:40798:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 3892 00000100:00080000:0.0:1787829092.198887:0:40798:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787829092.198891:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829092.198894:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787829092.198898:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787829092.198901:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787829092.198916:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:0.0:1787829092.198918:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:0.0:1787829092.198919:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829092.198920:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:0.0:1787829092.198924:0:40798:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787829092.198926:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829092.198928:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787829092.198932:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787829092.198935:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829092.198937:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1787829092.198939:0:40798:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829092.198954:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787829092.198955:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829092.198958:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787829092.198961:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787829092.198965:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829092.198980:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787829092.198988:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d24000 x1874670646795136/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787829092.198998:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787829092.199002:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829092.199015:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787829092.199018:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829092.199020:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1787829092.199022:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787829092.199023:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787829092.199025:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787829092.199026:0:40798:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3892 00000100:00080000:0.0:1787829092.199029:0:40798:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787829092.199030:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829092.199031:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787829092.199033:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787829092.199034:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787829092.199036:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000020:00000001:0.0:1787829092.199038:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:0.0:1787829092.199039:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829092.199040:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:0.0:1787829092.199042:0:40798:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787829092.199043:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829092.199045:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787829092.199051:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787829092.199053:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829092.199054:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:1.0:1787829092.199058:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000008:00100000:0.0:1787829092.199059:0:40798:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1787829092.199064:0:40798:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829092.199066:0:40798:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829092.199066:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 02000000:00000001:0.0:1787829092.199069:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787829092.199072:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829092.199072:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1787829092.199077:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787829092.199078:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:1.0:1787829092.199078:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 02000000:00000001:0.0:1787829092.199080:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829092.199084:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829092.199085:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787829092.199093:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d270c0 x1874670646795264/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:1.0:1787829092.199093:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787829092.199100:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787829092.199101:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787829092.199104:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829092.199107:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787829092.199109:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829092.199111:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1787829092.199112:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787829092.199113:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787829092.199115:0:40798:0:(import.c:485:import_select_connection()) Process entered 02000000:00000001:1.0:1787829092.199115:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787829092.199120:0:40798:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3892 00000100:00080000:0.0:1787829092.199122:0:40798:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787829092.199125:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829092.199127:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00100000:1.0:1787829092.199127:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d24000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646795136:192.168.203.143@tcp:250:kworker.0 00000020:00000001:0.0:1787829092.199131:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787829092.199133:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787829092.199136:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:1.0:1787829092.199138:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000020:00000001:0.0:1787829092.199139:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:0.0:1787829092.199143:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829092.199144:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 02000000:00000001:1.0:1787829092.199146:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000100:00080000:0.0:1787829092.199148:0:40798:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787829092.199151:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829092.199152:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829092.199153:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787829092.199156:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829092.199158:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1787829092.199159:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829092.199163:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1787829092.199164:0:40798:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1787829092.199166:0:40798:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829092.199167:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829092.199168:0:40798:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829092.199174:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787829092.199175:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829092.199176:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787829092.199177:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787829092.199178:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829092.199180:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787829092.199183:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d25a40 x1874670646795392/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787829092.199188:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787829092.199190:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829092.199192:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787829092.199194:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829092.199196:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1787829092.199196:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787829092.199197:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787829092.199199:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:1.0:1787829092.199199:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00080000:0.0:1787829092.199204:0:40798:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3892 00000100:00000001:1.0:1787829092.199204:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601336320 : -126234108215296 : ffff8d30d4d24000) 00000100:00080000:0.0:1787829092.199208:0:40798:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787829092.199210:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829092.199211:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787829092.199213:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:1.0:1787829092.199213:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000020:00000001:0.0:1787829092.199217:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787829092.199219:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000020:00000001:0.0:1787829092.199220:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:0.0:1787829092.199221:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829092.199222:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:0.0:1787829092.199224:0:40798:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787829092.199225:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829092.199227:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787829092.199228:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787829092.199229:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829092.199230:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1787829092.199232:0:40798:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1787829092.199234:0:40798:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829092.199235:0:40798:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829092.199238:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787829092.199239:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829092.199239:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787829092.199240:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787829092.199241:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829092.199246:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787829092.199252:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d24f00 x1874670646795520/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787829092.199259:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787829092.199261:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829092.199263:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829092.199275:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829092.199282:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4298566400 00000100:00000001:1.0:1787829092.199296:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 00000100:00000001:1.0:1787829092.199303:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4298566400 00000100:00000001:1.0:1787829092.199310:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 00000100:00000001:1.0:1787829092.199314:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829092.199317:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829092.199322:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829092.199325:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829092.199335:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787829092.199338:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:1.0:1787829092.199341:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787829092.199343:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1787829092.199345:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787829092.199347:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787829092.199349:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829092.199352:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787829092.199354:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829092.199356:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829092.199361:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d270c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646795264:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787829092.199366:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787829092.199368:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787829092.199369:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829092.199371:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787829092.199375:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829092.199383:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787829092.199385:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601348800 : -126234108202816 : ffff8d30d4d270c0) 00000100:00000001:1.0:1787829092.199390:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:1.0:1787829092.199406:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829092.199413:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298565142 00000100:00000001:1.0:1787829092.199416:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298566400 00000100:00000001:1.0:1787829092.199419:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298565142 00000100:00000001:1.0:1787829092.199421:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298566400 00000100:00000001:1.0:1787829092.199424:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829092.199426:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829092.199428:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787829092.199430:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787829092.199431:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829092.199434:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787829092.199436:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829092.199437:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829092.199442:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d25a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646795392:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787829092.199447:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787829092.199449:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787829092.199451:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829092.199454:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787829092.199477:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829092.199485:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787829092.199487:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601343040 : -126234108208576 : ffff8d30d4d25a40) 00000100:00000001:1.0:1787829092.199491:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:1.0:1787829092.199503:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829092.199506:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298565142 00000100:00000001:1.0:1787829092.199509:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298566400 00000100:00000001:1.0:1787829092.199512:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298565142 00000100:00000001:1.0:1787829092.199514:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298566400 00000100:00000001:1.0:1787829092.199517:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829092.199518:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829092.199520:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787829092.199521:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787829092.199522:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829092.199524:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787829092.199525:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829092.199527:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829092.199530:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d24f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646795520:192.168.203.143@tcp:8:kworker.0 00000100:00000001:1.0:1787829092.199532:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787829092.199533:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787829092.199535:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829092.199536:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787829092.199538:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829092.199543:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787829092.199544:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601340160 : -126234108211456 : ffff8d30d4d24f00) 00000100:00000001:1.0:1787829092.199546:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:1.0:1787829092.199553:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829092.199555:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298565155 00000100:00000001:1.0:1787829092.199557:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298566400 00000100:00000001:1.0:1787829092.199559:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298565155 00000100:00000001:1.0:1787829092.199560:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298566400 00000100:00000001:1.0:1787829092.199561:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829092.199563:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829092.199565:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829092.199566:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829092.199569:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787829092.199570:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:1.0:1787829092.199571:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787829092.199573:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1787829092.199574:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829092.199576:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829093.094663:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829093.094670:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829093.094729:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829093.094733:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829093.094737:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829094.118708:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829094.118715:0:815:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1787829094.118717:0:815:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1787829094.118719:0:815:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829094.118722:0:815:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1787829094.118724:0:815:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829094.118725:0:815:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829094.118727:0:815:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829094.118802:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829094.118805:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829094.118808:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829095.142691:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829095.142696:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829095.142757:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829095.142761:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829095.142765:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:3.0:1787829095.270885:0:2379:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.143:988 00000400:00000100:3.0:1787829095.270951:0:2379:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.143@tcp at host 192.168.203.143 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:3.0:1787829095.270966:0:2379:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.143@tcp 00000100:00000001:3.0:1787829095.270978:0:2379:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787829095.270999:0:2379:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829095.271002:0:2379:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829095.271005:0:2379:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:3.0:1787829095.271014:0:2379:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.143@tcp 00000100:00000001:3.0:1787829095.271020:0:2379:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787829095.271026:0:2379:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829095.271027:0:2379:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829095.271029:0:2379:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:3.0:1787829095.271035:0:2379:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.143@tcp 00000100:00000001:3.0:1787829095.271041:0:2379:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1787829095.271043:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1787829095.271044:0:2379:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829095.271059:0:2379:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829095.271061:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787829095.271061:0:2379:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:3.0:1787829095.271067:0:2379:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.143@tcp 00000100:00000001:0.0:1787829095.271072:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00000001:3.0:1787829095.271075:0:2379:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787829095.271081:0:2379:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829095.271085:0:2379:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787829095.271087:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829092/real 1787829095] req@ffff8d30d4d270c0 x1874670646795264/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829117 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829095.271088:0:2379:0:(events.c:68:request_out_callback()) Process leaving 00000400:00000001:0.0:1787829095.271103:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787829095.271106:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787829095.271124:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787829095.271127:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787829095.271129:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829095.271132:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787829095.271134:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787829095.271153:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d270c0 x1874670646795264/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829117 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787829095.271160:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829095.271163:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787829095.271166:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829095.271168:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787829095.271169:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829095.271172:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787829095.271175:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787829095.271179:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829095.271185:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787829095.271188:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787829095.271192:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d270c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646795264:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787829095.271197:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787829095.271201:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829092/real 1787829095] req@ffff8d30d4d25a40 x1874670646795392/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829117 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787829095.271209:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787829095.271210:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787829095.271214:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787829095.271215:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787829095.271217:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829095.271219:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787829095.271220:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787829095.271224:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d25a40 x1874670646795392/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829117 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787829095.271231:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829095.271233:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787829095.271236:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829095.271238:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787829095.271239:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829095.271240:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787829095.271242:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787829095.271245:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829095.271248:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787829095.271251:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787829095.271254:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d25a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646795392:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787829095.271259:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787829095.271263:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829092/real 1787829095] req@ffff8d30d4d24f00 x1874670646795520/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829117 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787829095.271269:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787829095.271270:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787829095.271273:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787829095.271274:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787829095.271275:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829095.271277:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787829095.271278:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787829095.271282:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d24f00 x1874670646795520/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829117 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787829095.271305:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829095.271308:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787829095.271310:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829095.271311:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787829095.271313:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829095.271314:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787829095.271316:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787829095.271319:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829095.271323:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787829095.271325:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787829095.271329:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d24f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646795520:192.168.203.143@tcp:8:kworker.0 00000100:00000001:0.0:1787829095.271333:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787829095.271337:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829092/real 1787829095] req@ffff8d30d4d24000 x1874670646795136/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829117 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787829095.271345:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787829095.271346:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787829095.271355:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787829095.271357:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787829095.271358:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829095.271361:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787829095.271362:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787829095.271366:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d24000 x1874670646795136/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829117 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787829095.271373:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829095.271376:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787829095.271378:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829095.271379:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787829095.271380:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829095.271381:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787829095.271383:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787829095.271385:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829095.271387:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787829095.271388:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787829095.271390:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d24000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646795136:192.168.203.143@tcp:250:kworker.0 00000100:00000001:0.0:1787829095.271394:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829095.271397:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787829095.271399:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787829095.271403:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787829095.271409:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787829095.271412:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787829095.271414:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787829095.271418:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787829095.271421:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787829095.271424:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787829095.271426:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829095.271428:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787829095.271429:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787829095.271430:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787829095.271432:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787829095.271433:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787829095.271448:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787829095.271451:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787829095.271452:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787829095.271454:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787829095.271455:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829095.271457:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787829095.271459:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787829095.271460:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787829095.271463:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787829095.271464:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787829095.271465:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787829095.271467:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787829095.271469:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787829095.271472:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787829095.271473:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829095.271475:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787829095.271476:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787829095.271478:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787829095.271480:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787829095.271481:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787829095.271482:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787829095.271484:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787829095.271485:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787829095.271486:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787829095.271488:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829095.271490:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829095.271499:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787829095.271501:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829095.271504:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787829095.271505:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829096.166716:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829096.166721:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829096.166778:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829096.166781:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829096.166784:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829097.190696:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829097.190704:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829097.190751:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829097.190755:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829097.190758:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787829097.318692:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829097.318702:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1787829097.318705:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787829097.318709:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787829097.318713:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787829097.318722:0:40798:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 3897 00000100:00080000:0.0:1787829097.318733:0:40798:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787829097.318737:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829097.318740:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787829097.318744:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787829097.318747:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787829097.318750:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:0.0:1787829097.318753:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:0.0:1787829097.318755:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829097.318757:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:0.0:1787829097.318762:0:40798:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787829097.318765:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829097.318768:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787829097.318772:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787829097.318775:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829097.318777:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1787829097.318781:0:40798:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829097.318789:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787829097.318791:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829097.318794:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787829097.318797:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787829097.318801:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829097.318812:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787829097.318821:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d24000 x1874670646795648/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787829097.318833:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787829097.318837:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829097.318857:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787829097.318860:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829097.318863:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1787829097.318865:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787829097.318867:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787829097.318870:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787829097.318872:0:40798:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3897 00000100:00080000:0.0:1787829097.318876:0:40798:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787829097.318879:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829097.318881:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787829097.318884:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787829097.318886:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787829097.318888:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000020:00000001:0.0:1787829097.318891:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:0.0:1787829097.318893:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829097.318895:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:0.0:1787829097.318899:0:40798:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:1.0:1787829097.318900:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787829097.318902:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829097.318906:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1787829097.318907:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1787829097.318911:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 02000000:00000001:0.0:1787829097.318913:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829097.318914:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 02000000:00000001:0.0:1787829097.318919:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829097.318919:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829097.318925:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 02000000:00000001:1.0:1787829097.318926:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000008:00100000:0.0:1787829097.318929:0:40798:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 02000000:00000001:1.0:1787829097.318931:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829097.318936:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:0.0:1787829097.318940:0:40798:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829097.318943:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d24000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646795648:192.168.203.143@tcp:250:kworker.0 00000100:00000001:0.0:1787829097.318945:0:40798:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829097.318952:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:1.0:1787829097.318952:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787829097.318956:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829097.318957:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:0.0:1787829097.318958:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787829097.318961:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1787829097.318962:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829097.318964:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829097.318968:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:0.0:1787829097.318972:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829097.318975:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787829097.318981:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d24f00 x1874670646795776/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787829097.318992:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787829097.318994:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829097.318996:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787829097.318999:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829097.319000:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601336320 : -126234108215296 : ffff8d30d4d24000) 00000100:00080000:0.0:1787829097.319003:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1787829097.319008:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00080000:0.0:1787829097.319009:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1787829097.319013:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787829097.319015:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787829097.319018:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787829097.319021:0:40798:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3897 00000100:00080000:0.0:1787829097.319042:0:40798:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787829097.319045:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829097.319047:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787829097.319051:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000800:00000100:1.0:1787829097.319053:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000020:00000001:0.0:1787829097.319059:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787829097.319061:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000020:00000001:0.0:1787829097.319065:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:1.0:1787829097.319066:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1787829097.319068:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829097.319072:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:1.0:1787829097.319073:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829097.319078:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787829097.319081:0:40798:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:1.0:1787829097.319083:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787829097.319088:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829097.319092:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1787829097.319092:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829097.319095:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829097.319099:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4298571520 02000000:00000001:0.0:1787829097.319100:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829097.319106:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1787829097.319109:0:40798:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00000001:1.0:1787829097.319112:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 00000008:00000001:0.0:1787829097.319113:0:40798:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829097.319118:0:40798:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829097.319121:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4298571520 02000000:00000001:0.0:1787829097.319124:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787829097.319128:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829097.319128:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 00000100:00000001:1.0:1787829097.319133:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787829097.319134:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:1.0:1787829097.319136:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829097.319139:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:1.0:1787829097.319140:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 02000000:00000001:0.0:1787829097.319144:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829097.319147:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829097/real 1787829097] req@ffff8d30d4d24000 x1874670646795648/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829122 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787829097.319152:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:1.0:1787829097.319162:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00100000:0.0:1787829097.319163:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d25a40 x1874670646795904/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000001:1.0:1787829097.319165:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787829097.319171:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787829097.319175:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787829097.319176:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787829097.319179:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829097.319181:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829097.319184:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787829097.319185:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829097.319188:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:0.0:1787829097.319190:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829097.319195:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1787829097.319197:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00100000:1.0:1787829097.319198:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d24000 x1874670646795648/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829122 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829097.319200:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787829097.319205:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:1.0:1787829097.319211:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:0.0:1787829097.319218:0:40798:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3897 00000100:00000001:1.0:1787829097.319218:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787829097.319225:0:40798:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1787829097.319225:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829097.319229:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787829097.319230:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787829097.319233:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829097.319234:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787829097.319239:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:1.0:1787829097.319239:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000020:00000001:0.0:1787829097.319244:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:1.0:1787829097.319244:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000020:00000001:0.0:1787829097.319246:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00080000:1.0:1787829097.319251:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000020:00000001:0.0:1787829097.319252:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:0.0:1787829097.319256:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829097.319259:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:1.0:1787829097.319262:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00080000:0.0:1787829097.319266:0:40798:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:1.0:1787829097.319268:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000001:0.0:1787829097.319270:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829097.319274:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00100000:1.0:1787829097.319277:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d24000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646795648:192.168.203.143@tcp:250:kworker.0 02000000:00000001:0.0:1787829097.319278:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787829097.319285:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829097.319287:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829097.319289:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:1.0:1787829097.319293:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000008:00100000:0.0:1787829097.319294:0:40798:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00000001:1.0:1787829097.319297:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 00000008:00000001:0.0:1787829097.319302:0:40798:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829097.319302:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 00000100:00000001:0.0:1787829097.319306:0:40798:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829097.319308:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787829097.319312:0:2390:0:(genops.c:1217:class_import_put()) Process entered 02000000:00000001:0.0:1787829097.319313:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000020:00000001:1.0:1787829097.319315:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787829097.319317:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829097.319321:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1787829097.319321:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787829097.319326:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1787829097.319326:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 02000000:00000001:0.0:1787829097.319331:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829097.319331:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787829097.319334:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829097.319337:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829097.319338:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787829097.319348:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d270c0 x1874670646796032/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787829097.319348:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787829097.319352:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829097.319357:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787829097.319360:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1787829097.319362:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1787829097.319364:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787829097.319366:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787829097.319370:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829097.319370:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829097.319376:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:0.0:1787829097.319379:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829097.319379:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829097.319385:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829097.319390:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d24f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646795776:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787829097.319394:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787829097.319396:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787829097.319398:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829097.319400:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787829097.319403:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829097.319411:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787829097.319412:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601340160 : -126234108211456 : ffff8d30d4d24f00) 00000100:00000001:1.0:1787829097.319417:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1787829097.319446:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787829097.319452:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787829097.319454:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829097.319456:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829097.319457:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787829097.319460:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829097.319464:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298570262 00000100:00000001:1.0:1787829097.319467:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298571520 00000100:00000001:1.0:1787829097.319471:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298570262 00000100:00000001:1.0:1787829097.319473:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298571520 00000100:00000001:1.0:1787829097.319475:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829097.319476:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829097.319479:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787829097.319484:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829097/real 1787829097] req@ffff8d30d4d24f00 x1874670646795776/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829122 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787829097.319492:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787829097.319493:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787829097.319495:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787829097.319497:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787829097.319498:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829097.319500:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829097.319501:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787829097.319504:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d24f00 x1874670646795776/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829122 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787829097.319512:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829097.319514:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787829097.319516:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829097.319518:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829097.319519:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829097.319521:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787829097.319522:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787829097.319524:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829097.319527:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787829097.319530:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787829097.319532:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d24f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646795776:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787829097.319536:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787829097.319537:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787829097.319539:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829097.319540:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787829097.319542:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829097.319544:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829097.319546:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d25a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646795904:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787829097.319549:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787829097.319551:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787829097.319552:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829097.319554:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787829097.319557:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829097.319562:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787829097.319564:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601343040 : -126234108208576 : ffff8d30d4d25a40) 00000100:00000001:1.0:1787829097.319568:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1787829097.319581:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787829097.319585:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787829097.319588:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829097.319589:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829097.319590:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787829097.319593:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829097.319597:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298570262 00000100:00000001:1.0:1787829097.319635:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298571520 00000100:00000001:1.0:1787829097.319638:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298570263 00000100:00000001:1.0:1787829097.319640:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298571520 00000100:00000001:1.0:1787829097.319642:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829097.319644:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829097.319646:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787829097.319650:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829097/real 1787829097] req@ffff8d30d4d25a40 x1874670646795904/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829122 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787829097.319658:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787829097.319659:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787829097.319662:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787829097.319664:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787829097.319666:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829097.319668:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829097.319669:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787829097.319673:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d25a40 x1874670646795904/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829122 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787829097.319679:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829097.319681:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787829097.319683:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829097.319685:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829097.319686:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829097.319688:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787829097.319690:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787829097.319692:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829097.319695:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787829097.319697:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787829097.319700:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d25a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646795904:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787829097.319704:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829097.319705:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829097.319706:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787829097.319708:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787829097.319710:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787829097.319712:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787829097.319713:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787829097.319715:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787829097.319716:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787829097.319719:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787829097.319720:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829097.319722:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829097.319723:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787829097.319724:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787829097.319726:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787829097.319727:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787829097.319728:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787829097.319730:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787829097.319731:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787829097.319732:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787829097.319734:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829097.319736:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829097.319739:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787829097.319741:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829097.319742:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787829097.319744:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1787829097.319746:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787829097.319747:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787829097.319748:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829097.319750:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787829097.319752:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829097.319754:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829097.319757:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d270c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646796032:192.168.203.143@tcp:8:kworker.0 00000100:00000001:1.0:1787829097.319760:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787829097.319762:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787829097.319764:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829097.319765:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787829097.319768:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829097.319774:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787829097.319776:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601348800 : -126234108202816 : ffff8d30d4d270c0) 00000100:00000001:1.0:1787829097.319779:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1787829097.319791:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787829097.319796:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787829097.319799:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829097.319800:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829097.319802:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787829097.319804:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829097.319809:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298570276 00000100:00000001:1.0:1787829097.319811:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298571520 00000100:00000001:1.0:1787829097.319813:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298570276 00000100:00000001:1.0:1787829097.319816:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298571520 00000100:00000001:1.0:1787829097.319818:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829097.319820:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829097.319821:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787829097.319825:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829097/real 1787829097] req@ffff8d30d4d270c0 x1874670646796032/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829122 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787829097.319833:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787829097.319835:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787829097.319837:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787829097.319839:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787829097.319840:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829097.319842:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829097.319844:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787829097.319848:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d270c0 x1874670646796032/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829122 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787829097.319854:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829097.319856:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787829097.319858:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829097.319860:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829097.319861:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829097.319863:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787829097.319865:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787829097.319868:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829097.319871:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787829097.319873:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787829097.319877:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d270c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646796032:192.168.203.143@tcp:8:kworker.0 00000100:00000001:1.0:1787829097.319887:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829097.319889:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829097.319891:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787829097.319892:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787829097.319894:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787829097.319896:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787829097.319897:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787829097.319899:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787829097.319901:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787829097.319904:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787829097.319906:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829097.319908:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829097.319911:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787829097.319913:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829097.319915:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787829097.319917:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829098.214676:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829098.214683:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829098.214734:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829098.214738:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829098.214741:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829099.238712:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829099.238717:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829099.238775:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829099.238778:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829099.238781:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829100.262683:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829100.262689:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829100.262738:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829100.262741:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829100.262744:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829101.286717:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829101.286722:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829101.286773:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829101.286777:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829101.286780:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829102.310673:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829102.310678:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829102.310734:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829102.310738:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829102.310741:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787829102.438692:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829102.438700:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1787829102.438702:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1787829102.438706:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787829102.438711:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1787829102.438719:0:468:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 3902 00000100:00080000:1.0:1787829102.438731:0:468:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1787829102.438734:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787829102.438736:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:1.0:1787829102.438741:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:1.0:1787829102.438744:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:1.0:1787829102.438748:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:1.0:1787829102.438750:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:1.0:1787829102.438753:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787829102.438754:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:1.0:1787829102.438758:0:468:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:1.0:1787829102.438760:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829102.438762:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1787829102.438765:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829102.438767:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829102.438769:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:1.0:1787829102.438771:0:468:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829102.438782:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1787829102.438783:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829102.438786:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1787829102.438788:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1787829102.438792:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829102.438802:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829102.438809:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d13ecb40 x1874670646796160/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787829102.438817:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787829102.438820:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829102.438842:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1787829102.438845:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829102.438847:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1787829102.438848:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1787829102.438850:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787829102.438852:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1787829102.438855:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3902 00000100:00080000:1.0:1787829102.438858:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1787829102.438860:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787829102.438861:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:1.0:1787829102.438864:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:1.0:1787829102.438865:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:1.0:1787829102.438866:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000020:00000001:1.0:1787829102.438868:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:0.0:1787829102.438869:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787829102.438870:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787829102.438873:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:0.0:1787829102.438876:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00080000:1.0:1787829102.438877:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:1.0:1787829102.438881:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829102.438882:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 02000000:00000001:1.0:1787829102.438883:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:0.0:1787829102.438887:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 02000000:00000001:1.0:1787829102.438890:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787829102.438892:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829102.438893:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829102.438896:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 02000000:00000001:0.0:1787829102.438899:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000008:00100000:1.0:1787829102.438900:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 02000000:00000001:0.0:1787829102.438903:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000008:00000001:1.0:1787829102.438904:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829102.438909:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829102.438911:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829102.438913:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1787829102.438916:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829102.438918:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00100000:0.0:1787829102.438920:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d13ecb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646796160:192.168.203.143@tcp:250:kworker.0 02000000:00000001:1.0:1787829102.438920:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1787829102.438923:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829102.438929:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:1.0:1787829102.438929:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829102.438936:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000100:00100000:1.0:1787829102.438937:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d13ed680 x1874670646796288/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:0.0:1787829102.438940:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829102.438946:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:1.0:1787829102.438947:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787829102.438950:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829102.438952:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829102.438953:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1787829102.438954:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829102.438959:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:1.0:1787829102.438961:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1787829102.438962:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787829102.438963:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1787829102.438965:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3902 00000100:00080000:1.0:1787829102.438968:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1787829102.438969:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787829102.438970:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:1.0:1787829102.438971:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:0.0:1787829102.438972:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000020:00000001:1.0:1787829102.438973:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:1.0:1787829102.438976:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:0.0:1787829102.438977:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839541340992 : -126234168210624 : ffff8d30d13ecb40) 00000020:00000001:1.0:1787829102.438978:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:1.0:1787829102.438983:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787829102.438985:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:0.0:1787829102.438987:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00080000:1.0:1787829102.438987:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:1.0:1787829102.438991:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829102.438993:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1787829102.438995:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829102.438997:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829102.438998:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:1.0:1787829102.438999:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1787829102.439001:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829102.439002:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829102.439005:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1787829102.439006:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829102.439007:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1787829102.439008:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1787829102.439009:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829102.439012:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829102.439015:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d13ed2c0 x1874670646796416/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787829102.439019:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787829102.439021:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829102.439022:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1787829102.439023:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829102.439025:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1787829102.439026:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1787829102.439027:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787829102.439029:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1787829102.439031:0:468:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3902 00000100:00080000:1.0:1787829102.439034:0:468:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1787829102.439038:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000800:00000100:0.0:1787829102.439039:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787829102.439039:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:1.0:1787829102.439043:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:1.0:1787829102.439044:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:1.0:1787829102.439045:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000020:00000001:1.0:1787829102.439047:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:1.0:1787829102.439049:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787829102.439050:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:0.0:1787829102.439054:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00080000:1.0:1787829102.439054:0:468:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:1.0:1787829102.439057:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829102.439060:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:0.0:1787829102.439062:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 02000000:00000001:1.0:1787829102.439062:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829102.439066:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829102.439067:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829102.439068:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:1.0:1787829102.439071:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00000001:0.0:1787829102.439072:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000008:00000001:1.0:1787829102.439074:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829102.439077:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829102.439080:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:0.0:1787829102.439081:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829102.439082:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829102.439087:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1787829102.439088:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1787829102.439089:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829102.439090:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4298576640 00000100:00000001:1.0:1787829102.439094:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829102.439098:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d13efc00 x1874670646796544/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787829102.439101:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 00000100:00000001:1.0:1787829102.439104:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787829102.439106:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829102.439108:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829102.439111:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4298576640 00000100:00000001:0.0:1787829102.439117:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1787829102.439123:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787829102.439126:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829102.439128:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787829102.439136:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829102/real 1787829102] req@ffff8d30d13ecb40 x1874670646796160/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829127 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787829102.439147:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787829102.439149:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787829102.439152:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787829102.439155:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787829102.439156:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829102.439159:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787829102.439161:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787829102.439165:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d13ecb40 x1874670646796160/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829127 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787829102.439172:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829102.439175:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787829102.439178:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829102.439180:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787829102.439182:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829102.439185:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787829102.439187:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787829102.439191:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829102.439195:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787829102.439200:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787829102.439204:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d13ecb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646796160:192.168.203.143@tcp:250:kworker.0 00000100:00000001:0.0:1787829102.439209:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829102.439212:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787829102.439215:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787829102.439217:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787829102.439220:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787829102.439222:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787829102.439224:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787829102.439228:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787829102.439230:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787829102.439233:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787829102.439235:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829102.439237:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829102.439244:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787829102.439246:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829102.439248:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787829102.439250:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787829102.439251:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1787829102.439252:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787829102.439254:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829102.439256:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787829102.439258:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787829102.439261:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787829102.439264:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d13ed680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646796288:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787829102.439268:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787829102.439270:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787829102.439272:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829102.439274:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1787829102.439277:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829102.439283:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787829102.439284:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839541343872 : -126234168207744 : ffff8d30d13ed680) 00000100:00000001:0.0:1787829102.439289:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1787829102.439302:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:0.0:1787829102.439306:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1787829102.439308:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787829102.439310:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829102.439311:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787829102.439314:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829102.439319:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298575382 00000100:00000001:0.0:1787829102.439322:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298576640 00000100:00000001:0.0:1787829102.439324:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298575382 00000100:00000001:0.0:1787829102.439327:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298576640 00000100:00000001:0.0:1787829102.439329:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787829102.439331:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829102.439333:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787829102.439336:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829102/real 1787829102] req@ffff8d30d13ed680 x1874670646796288/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829127 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787829102.439345:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787829102.439346:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787829102.439348:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787829102.439350:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787829102.439351:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829102.439353:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787829102.439354:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787829102.439358:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d13ed680 x1874670646796288/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829127 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787829102.439365:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829102.439367:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787829102.439370:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829102.439371:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787829102.439373:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829102.439375:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787829102.439377:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787829102.439380:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829102.439383:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787829102.439402:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787829102.439406:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d13ed680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646796288:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787829102.439411:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1787829102.439412:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787829102.439414:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829102.439417:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787829102.439419:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787829102.439421:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787829102.439424:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d13ed2c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646796416:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787829102.439427:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787829102.439429:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787829102.439431:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829102.439433:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1787829102.439435:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829102.439440:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787829102.439441:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839541342912 : -126234168208704 : ffff8d30d13ed2c0) 00000100:00000001:0.0:1787829102.439445:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1787829102.439456:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:0.0:1787829102.439460:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1787829102.439463:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787829102.439464:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829102.439466:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787829102.439469:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829102.439472:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298575382 00000100:00000001:0.0:1787829102.439474:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298576640 00000100:00000001:0.0:1787829102.439476:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298575382 00000100:00000001:0.0:1787829102.439478:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298576640 00000100:00000001:0.0:1787829102.439480:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787829102.439482:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829102.439484:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787829102.439487:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829102/real 1787829102] req@ffff8d30d13ed2c0 x1874670646796416/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829127 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787829102.439495:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787829102.439496:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787829102.439498:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787829102.439500:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787829102.439501:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829102.439504:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787829102.439505:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787829102.439509:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d13ed2c0 x1874670646796416/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829127 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787829102.439516:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829102.439518:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787829102.439521:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829102.439523:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787829102.439524:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829102.439526:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787829102.439527:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787829102.439530:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829102.439534:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787829102.439536:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787829102.439539:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d13ed2c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646796416:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787829102.439543:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1787829102.439545:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787829102.439546:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829102.439548:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787829102.439550:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787829102.439552:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787829102.439555:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d13efc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646796544:192.168.203.143@tcp:8:kworker.0 00000100:00000001:0.0:1787829102.439558:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787829102.439560:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787829102.439562:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829102.439564:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1787829102.439566:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829102.439571:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787829102.439572:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839541353472 : -126234168198144 : ffff8d30d13efc00) 00000100:00000001:0.0:1787829102.439575:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1787829102.439586:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:0.0:1787829102.439590:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1787829102.439592:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787829102.439594:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829102.439596:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787829102.439599:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829102.439623:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298575396 00000100:00000001:0.0:1787829102.439626:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298576640 00000100:00000001:0.0:1787829102.439628:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298575396 00000100:00000001:0.0:1787829102.439630:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298576640 00000100:00000001:0.0:1787829102.439633:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787829102.439635:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829102.439636:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787829102.439641:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829102/real 1787829102] req@ffff8d30d13efc00 x1874670646796544/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829127 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787829102.439649:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787829102.439650:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787829102.439653:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787829102.439654:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787829102.439656:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829102.439657:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787829102.439659:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787829102.439663:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d13efc00 x1874670646796544/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829127 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787829102.439670:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829102.439672:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787829102.439674:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829102.439676:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787829102.439677:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829102.439679:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787829102.439681:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787829102.439683:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829102.439687:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787829102.439689:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787829102.439692:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d13efc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646796544:192.168.203.143@tcp:8:kworker.0 00000100:00000001:0.0:1787829102.439696:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829102.439698:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787829102.439699:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787829102.439701:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787829102.439703:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787829102.439704:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787829102.439706:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787829102.439708:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787829102.439710:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787829102.439712:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787829102.439713:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829102.439715:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787829102.439716:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787829102.439718:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787829102.439720:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787829102.439721:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787829102.439722:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787829102.439725:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787829102.439726:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787829102.439728:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787829102.439729:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829102.439731:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787829102.439732:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787829102.439733:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787829102.439735:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787829102.439737:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787829102.439738:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787829102.439740:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787829102.439741:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787829102.439743:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787829102.439744:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829102.439746:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829102.439749:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787829102.439751:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829102.439753:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787829102.439755:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829103.334724:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829103.334730:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829103.334809:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829103.334816:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829103.334821:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829104.358717:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829104.358722:0:815:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1787829104.358724:0:815:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1787829104.358727:0:815:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829104.358731:0:815:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1787829104.358733:0:815:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829104.358735:0:815:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829104.358737:0:815:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829104.358807:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829104.358811:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829104.358814:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829105.382706:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829105.382713:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829105.382772:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829105.382775:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829105.382778:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829106.406697:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829106.406702:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829106.406786:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829106.406789:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829106.406791:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829107.430732:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829107.430742:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829107.430797:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829107.430801:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829107.430805:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787829107.558707:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829107.558717:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1787829107.558720:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787829107.558724:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787829107.558729:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787829107.558736:0:40798:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 3908 00000100:00080000:0.0:1787829107.558748:0:40798:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787829107.558751:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829107.558753:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787829107.558756:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787829107.558759:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787829107.558762:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:0.0:1787829107.558763:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:0.0:1787829107.558765:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829107.558766:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:0.0:1787829107.558770:0:40798:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787829107.558772:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829107.558774:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787829107.558777:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787829107.558780:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829107.558782:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1787829107.558784:0:40798:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829107.558796:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787829107.558798:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829107.558801:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787829107.558804:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787829107.558808:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829107.558820:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787829107.558829:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d27c00 x1874670646796672/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787829107.558837:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787829107.558841:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829107.558864:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787829107.558867:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829107.558871:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1787829107.558873:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787829107.558876:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787829107.558880:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787829107.558885:0:40798:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3908 00000100:00080000:0.0:1787829107.558890:0:40798:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1787829107.558890:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787829107.558897:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787829107.558898:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787829107.558900:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:1.0:1787829107.558902:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787829107.558905:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000020:00000001:0.0:1787829107.558909:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:1.0:1787829107.558910:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787829107.558913:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787829107.558917:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 02000000:00000001:1.0:1787829107.558918:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000020:00000001:0.0:1787829107.558919:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:0.0:1787829107.558923:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 02000000:00000001:1.0:1787829107.558923:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787829107.558932:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 02000000:00000001:1.0:1787829107.558933:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787829107.558956:0:40798:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00100000:1.0:1787829107.558957:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d27c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646796672:192.168.203.143@tcp:250:kworker.0 00000100:00000001:0.0:1787829107.558965:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829107.558972:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1787829107.558972:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787829107.558983:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787829107.558985:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829107.558987:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829107.558990:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829107.558992:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:0.0:1787829107.558994:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1787829107.558999:0:40798:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 02000000:00000001:1.0:1787829107.559000:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:0.0:1787829107.559012:0:40798:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829107.559016:0:40798:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829107.559021:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787829107.559022:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829107.559024:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787829107.559025:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:1.0:1787829107.559026:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 02000000:00000001:0.0:1787829107.559029:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829107.559029:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601351680 : -126234108199936 : ffff8d30d4d27c00) 00000100:00000001:1.0:1787829107.559043:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:0.0:1787829107.559045:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787829107.559053:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d252c0 x1874670646796800/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787829107.559060:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787829107.559062:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829107.559065:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787829107.559068:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829107.559071:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1787829107.559073:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787829107.559074:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787829107.559077:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787829107.559083:0:40798:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3908 00000800:00000100:1.0:1787829107.559083:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787829107.559097:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00080000:0.0:1787829107.559107:0:40798:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1787829107.559107:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787829107.559116:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787829107.559117:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829107.559121:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:1.0:1787829107.559122:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000020:00000001:0.0:1787829107.559129:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787829107.559131:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:1.0:1787829107.559134:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787829107.559135:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000020:00000001:0.0:1787829107.559144:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:1.0:1787829107.559146:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4298581760 00000100:00000001:0.0:1787829107.559147:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829107.559150:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:0.0:1787829107.559154:0:40798:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:1.0:1787829107.559155:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1787829107.559159:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829107.559165:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787829107.559168:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787829107.559169:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829107.559171:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1787829107.559173:0:40798:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00000001:1.0:1787829107.559173:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4298581760 00000008:00000001:0.0:1787829107.559177:0:40798:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829107.559182:0:40798:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829107.559187:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:1.0:1787829107.559187:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 02000000:00000001:0.0:1787829107.559189:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829107.559193:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787829107.559195:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787829107.559196:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829107.559197:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787829107.559201:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829107.559204:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787829107.559206:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d25680 x1874670646796928/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787829107.559210:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00000001:0.0:1787829107.559218:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787829107.559221:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829107.559222:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829107/real 1787829107] req@ffff8d30d4d27c00 x1874670646796672/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829132 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787829107.559224:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787829107.559237:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829107.559252:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1787829107.559257:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787829107.559263:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000400:00000001:1.0:1787829107.559264:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000001:0.0:1787829107.559269:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000400:00000001:1.0:1787829107.559269:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787829107.559275:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00080000:0.0:1787829107.559276:0:40798:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3908 00000100:00080000:0.0:1787829107.559284:0:40798:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1787829107.559284:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787829107.559295:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787829107.559296:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829107.559298:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787829107.559306:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:1.0:1787829107.559307:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000020:00000001:0.0:1787829107.559308:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787829107.559310:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:1.0:1787829107.559313:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1787829107.559314:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:0.0:1787829107.559322:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829107.559324:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:0.0:1787829107.559331:0:40798:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00100000:1.0:1787829107.559333:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d27c00 x1874670646796672/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829132 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787829107.559334:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829107.559339:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787829107.559343:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787829107.559349:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829107.559363:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1787829107.559365:0:40798:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1787829107.559371:0:40798:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829107.559376:0:40798:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829107.559384:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787829107.559388:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829107.559390:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829107.559396:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787829107.559400:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787829107.559402:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829107.559404:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787829107.559407:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829107.559411:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787829107.559414:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d26940 x1874670646797056/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787829107.559417:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829107.559423:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829107.559433:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787829107.559435:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787829107.559436:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829107.559446:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829107.559447:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787829107.559453:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829107.559460:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787829107.559464:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787829107.559469:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d27c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646796672:192.168.203.143@tcp:250:kworker.0 00000100:00000001:1.0:1787829107.559475:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829107.559478:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829107.559489:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787829107.559493:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787829107.559498:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787829107.559501:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787829107.559503:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787829107.559507:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787829107.559509:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787829107.559513:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787829107.559515:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829107.559517:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829107.559524:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787829107.559527:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829107.559529:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787829107.559531:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1787829107.559533:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787829107.559534:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787829107.559536:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829107.559539:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787829107.559541:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829107.559544:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829107.559547:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d252c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646796800:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787829107.559552:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787829107.559554:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787829107.559556:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829107.559558:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787829107.559562:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829107.559569:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787829107.559571:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601341120 : -126234108210496 : ffff8d30d4d252c0) 00000100:00000001:1.0:1787829107.559575:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1787829107.559591:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787829107.559596:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787829107.559599:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829107.559624:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829107.559627:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787829107.559631:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829107.559636:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298580503 00000100:00000001:1.0:1787829107.559639:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298581760 00000100:00000001:1.0:1787829107.559641:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298580503 00000100:00000001:1.0:1787829107.559643:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298581760 00000100:00000001:1.0:1787829107.559646:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829107.559648:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829107.559650:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787829107.559655:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829107/real 1787829107] req@ffff8d30d4d252c0 x1874670646796800/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829132 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787829107.559664:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787829107.559666:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787829107.559669:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787829107.559670:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787829107.559672:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829107.559674:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829107.559675:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787829107.559680:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d252c0 x1874670646796800/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829132 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787829107.559688:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829107.559690:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787829107.559692:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829107.559694:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829107.559696:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829107.559698:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787829107.559700:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787829107.559703:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829107.559706:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787829107.559708:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787829107.559712:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d252c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646796800:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787829107.559717:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787829107.559718:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787829107.559720:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829107.559722:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787829107.559724:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829107.559726:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829107.559729:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d25680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646796928:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787829107.559733:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787829107.559736:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787829107.559738:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829107.559740:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787829107.559743:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829107.559748:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787829107.559750:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601342080 : -126234108209536 : ffff8d30d4d25680) 00000100:00000001:1.0:1787829107.559754:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1787829107.559767:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787829107.559771:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787829107.559774:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829107.559775:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829107.559777:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787829107.559780:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829107.559784:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298580503 00000100:00000001:1.0:1787829107.559787:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298581760 00000100:00000001:1.0:1787829107.559789:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298580503 00000100:00000001:1.0:1787829107.559791:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298581760 00000100:00000001:1.0:1787829107.559794:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829107.559796:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829107.559798:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787829107.559802:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829107/real 1787829107] req@ffff8d30d4d25680 x1874670646796928/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829132 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787829107.559811:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787829107.559812:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787829107.559815:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787829107.559817:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787829107.559818:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829107.559820:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829107.559822:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787829107.559826:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d25680 x1874670646796928/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829132 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787829107.559834:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829107.559836:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787829107.559839:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829107.559840:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829107.559842:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829107.559844:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787829107.559845:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787829107.559848:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829107.559851:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787829107.559853:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787829107.559857:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d25680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646796928:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787829107.559861:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787829107.559863:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787829107.559864:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829107.559867:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787829107.559869:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829107.559871:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829107.559874:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d26940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646797056:192.168.203.143@tcp:8:kworker.0 00000100:00000001:1.0:1787829107.559878:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787829107.559881:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787829107.559882:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829107.559884:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787829107.559887:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829107.559893:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787829107.559895:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601346880 : -126234108204736 : ffff8d30d4d26940) 00000100:00000001:1.0:1787829107.559898:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1787829107.559911:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787829107.559915:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787829107.559918:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829107.559919:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829107.559921:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787829107.559924:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829107.559927:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298580516 00000100:00000001:1.0:1787829107.559930:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298581760 00000100:00000001:1.0:1787829107.559932:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298580516 00000100:00000001:1.0:1787829107.559934:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298581760 00000100:00000001:1.0:1787829107.559936:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829107.559939:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829107.559940:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787829107.559945:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829107/real 1787829107] req@ffff8d30d4d26940 x1874670646797056/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829132 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787829107.559953:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787829107.559955:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787829107.559957:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787829107.559959:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787829107.559960:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829107.559962:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829107.559964:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787829107.559967:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d26940 x1874670646797056/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829132 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787829107.559981:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829107.559984:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787829107.559986:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829107.559988:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829107.559990:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829107.559992:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787829107.559993:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787829107.559996:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829107.560000:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787829107.560002:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787829107.560005:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d26940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646797056:192.168.203.143@tcp:8:kworker.0 00000100:00000001:1.0:1787829107.560009:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829107.560011:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829107.560012:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787829107.560014:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787829107.560016:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787829107.560018:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787829107.560020:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787829107.560022:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787829107.560023:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787829107.560027:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787829107.560029:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829107.560030:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829107.560032:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787829107.560033:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787829107.560035:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787829107.560036:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787829107.560037:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787829107.560039:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787829107.560040:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787829107.560042:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787829107.560044:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829107.560045:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829107.560046:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787829107.560048:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787829107.560050:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787829107.560051:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787829107.560052:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787829107.560054:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787829107.560055:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787829107.560058:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787829107.560059:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829107.560061:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829107.560064:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787829107.560066:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829107.560068:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787829107.560070:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829108.454710:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829108.454717:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829108.454777:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829108.454781:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829108.454784:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829109.478679:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829109.478686:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829109.478748:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829109.478752:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829109.478754:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829110.502696:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829110.502705:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829110.502758:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829110.502762:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829110.502764:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829111.526743:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829111.526760:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829111.526847:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829111.526851:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829111.526855:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829112.550731:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829112.550739:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829112.550798:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829112.550802:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829112.550806:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787829112.678765:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829112.678773:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1787829112.678776:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787829112.678781:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787829112.678786:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787829112.678793:0:40798:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 3913 00000100:00080000:2.0:1787829112.678804:0:40798:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787829112.678807:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829112.678810:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787829112.678814:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787829112.678816:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787829112.678819:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:2.0:1787829112.678822:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:2.0:1787829112.678823:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829112.678825:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:2.0:1787829112.678829:0:40798:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787829112.678831:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829112.678833:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787829112.678836:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787829112.678839:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829112.678842:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:2.0:1787829112.678845:0:40798:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829112.678855:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787829112.678856:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829112.678860:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787829112.678863:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787829112.678867:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829112.678879:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787829112.678889:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2e1c0 x1874670646797184/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787829112.678899:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787829112.678903:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829112.678912:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787829112.678915:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829112.678918:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1787829112.678920:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787829112.678922:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787829112.678924:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:1.0:1787829112.678927:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00080000:2.0:1787829112.678927:0:40798:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3913 00000100:00080000:2.0:1787829112.678932:0:40798:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787829112.678935:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787829112.678936:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1787829112.678936:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787829112.678941:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:1.0:1787829112.678942:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000020:00000001:2.0:1787829112.678942:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787829112.678945:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000020:00000001:2.0:1787829112.678947:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:2.0:1787829112.678948:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829112.678950:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:2.0:1787829112.678954:0:40798:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787829112.678958:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829112.678959:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 02000000:00000001:2.0:1787829112.678961:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1787829112.678964:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829112.678967:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787829112.678970:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829112.678972:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 02000000:00000001:1.0:1787829112.678973:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000008:00100000:2.0:1787829112.678977:0:40798:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 02000000:00000001:1.0:1787829112.678979:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000008:00000001:2.0:1787829112.678981:0:40798:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829112.678986:0:40798:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829112.678988:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829112.678991:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787829112.678993:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829112.678996:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787829112.678997:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00100000:1.0:1787829112.678998:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646797184:192.168.203.143@tcp:250:kworker.0 02000000:00000001:2.0:1787829112.678999:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829112.679006:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829112.679008:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00100000:2.0:1787829112.679012:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2d680 x1874670646797312/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:1.0:1787829112.679017:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:2.0:1787829112.679021:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 02000000:00000001:1.0:1787829112.679022:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829112.679025:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829112.679027:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:2.0:1787829112.679029:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787829112.679032:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 02000000:00000001:1.0:1787829112.679033:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787829112.679037:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1787829112.679039:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787829112.679040:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787829112.679041:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787829112.679044:0:40798:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3913 00000100:00080000:2.0:1787829112.679046:0:40798:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787829112.679047:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829112.679048:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787829112.679049:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787829112.679050:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787829112.679051:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000020:00000001:2.0:1787829112.679052:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:2.0:1787829112.679054:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829112.679056:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:2.0:1787829112.679058:0:40798:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:1.0:1787829112.679059:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787829112.679059:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829112.679060:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1787829112.679063:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417876928 : -126234291674688 : ffff8d30c9e2e1c0) 02000000:00000001:2.0:1787829112.679065:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787829112.679070:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829112.679072:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:2.0:1787829112.679072:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1787829112.679076:0:40798:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1787829112.679078:0:40798:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829112.679079:0:40798:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829112.679082:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787829112.679083:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829112.679085:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787829112.679085:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787829112.679086:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829112.679090:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787829112.679093:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2cb40 x1874670646797440/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787829112.679098:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787829112.679099:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829112.679101:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787829112.679103:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829112.679105:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1787829112.679106:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787829112.679107:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787829112.679108:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787829112.679110:0:40798:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3913 00000100:00080000:2.0:1787829112.679112:0:40798:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787829112.679113:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829112.679114:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787829112.679116:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787829112.679117:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787829112.679119:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000020:00000001:2.0:1787829112.679120:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:2.0:1787829112.679123:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829112.679126:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000800:00000100:1.0:1787829112.679127:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00080000:2.0:1787829112.679130:0:40798:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787829112.679133:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829112.679135:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787829112.679136:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787829112.679138:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829112.679140:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1787829112.679142:0:40798:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00000001:1.0:1787829112.679144:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000008:00000001:2.0:1787829112.679144:0:40798:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829112.679146:0:40798:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829112.679151:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787829112.679152:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829112.679154:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787829112.679154:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787829112.679157:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:1.0:1787829112.679160:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829112.679160:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829112.679166:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1787829112.679166:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787829112.679172:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2d2c0 x1874670646797568/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787829112.679174:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829112.679181:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787829112.679183:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4298586880 00000100:00000001:2.0:1787829112.679184:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829112.679190:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829112.679197:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 00000100:00000001:1.0:1787829112.679205:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4298586880 00000100:00000001:1.0:1787829112.679211:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 00000100:00000001:1.0:1787829112.679216:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829112.679219:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829112.679222:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787829112.679230:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829112/real 1787829112] req@ffff8d30c9e2e1c0 x1874670646797184/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829137 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787829112.679242:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787829112.679244:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787829112.679248:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787829112.679251:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787829112.679253:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829112.679256:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829112.679259:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787829112.679265:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2e1c0 x1874670646797184/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829137 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787829112.679274:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829112.679277:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787829112.679280:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829112.679282:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829112.679283:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829112.679285:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787829112.679287:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787829112.679290:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829112.679293:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787829112.679295:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787829112.679298:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646797184:192.168.203.143@tcp:250:kworker.0 00000100:00000001:1.0:1787829112.679303:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829112.679305:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829112.679307:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787829112.679310:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787829112.679326:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787829112.679329:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787829112.679331:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787829112.679335:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787829112.679337:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787829112.679342:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787829112.679344:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829112.679346:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829112.679354:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787829112.679356:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829112.679358:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787829112.679359:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1787829112.679361:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787829112.679362:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787829112.679363:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829112.679365:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787829112.679367:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829112.679370:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829112.679374:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2d680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646797312:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787829112.679378:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787829112.679381:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787829112.679384:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829112.679387:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787829112.679391:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829112.679400:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787829112.679402:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417874048 : -126234291677568 : ffff8d30c9e2d680) 00000100:00000001:1.0:1787829112.679408:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1787829112.679424:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787829112.679430:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787829112.679433:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829112.679435:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829112.679438:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787829112.679441:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829112.679446:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298585622 00000100:00000001:1.0:1787829112.679449:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298586880 00000100:00000001:1.0:1787829112.679452:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298585622 00000100:00000001:1.0:1787829112.679454:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298586880 00000100:00000001:1.0:1787829112.679456:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829112.679458:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829112.679460:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787829112.679465:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829112/real 1787829112] req@ffff8d30c9e2d680 x1874670646797312/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829137 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787829112.679471:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787829112.679472:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787829112.679474:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787829112.679476:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787829112.679477:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829112.679478:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829112.679479:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787829112.679482:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2d680 x1874670646797312/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829137 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787829112.679486:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829112.679488:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787829112.679489:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829112.679490:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829112.679491:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829112.679492:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787829112.679494:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787829112.679496:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829112.679503:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787829112.679504:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787829112.679507:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2d680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646797312:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787829112.679510:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787829112.679511:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787829112.679512:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829112.679514:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787829112.679515:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829112.679516:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829112.679519:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646797440:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787829112.679521:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787829112.679522:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787829112.679523:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829112.679525:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787829112.679527:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829112.679531:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787829112.679532:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417871168 : -126234291680448 : ffff8d30c9e2cb40) 00000100:00000001:1.0:1787829112.679535:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1787829112.679543:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787829112.679546:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787829112.679548:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829112.679549:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829112.679550:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787829112.679552:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829112.679554:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298585622 00000100:00000001:1.0:1787829112.679556:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298586880 00000100:00000001:1.0:1787829112.679558:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298585622 00000100:00000001:1.0:1787829112.679559:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298586880 00000100:00000001:1.0:1787829112.679561:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829112.679562:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829112.679563:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787829112.679565:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829112/real 1787829112] req@ffff8d30c9e2cb40 x1874670646797440/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829137 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787829112.679575:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787829112.679576:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787829112.679578:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787829112.679579:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787829112.679580:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829112.679581:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829112.679581:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787829112.679584:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2cb40 x1874670646797440/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829137 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787829112.679588:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829112.679590:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787829112.679591:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829112.679592:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829112.679593:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829112.679595:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787829112.679596:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787829112.679597:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829112.679599:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787829112.679618:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787829112.679621:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646797440:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787829112.679624:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787829112.679625:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787829112.679627:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829112.679628:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787829112.679630:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829112.679631:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829112.679634:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2d2c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646797568:192.168.203.143@tcp:8:kworker.0 00000100:00000001:1.0:1787829112.679637:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787829112.679638:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787829112.679640:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829112.679641:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787829112.679643:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829112.679647:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787829112.679649:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417873088 : -126234291678528 : ffff8d30c9e2d2c0) 00000100:00000001:1.0:1787829112.679652:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1787829112.679661:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787829112.679665:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787829112.679667:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829112.679669:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829112.679670:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787829112.679673:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829112.679675:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298585636 00000100:00000001:1.0:1787829112.679677:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298586880 00000100:00000001:1.0:1787829112.679680:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298585636 00000100:00000001:1.0:1787829112.679682:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298586880 00000100:00000001:1.0:1787829112.679684:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829112.679685:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829112.679686:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787829112.679689:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829112/real 1787829112] req@ffff8d30c9e2d2c0 x1874670646797568/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829137 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787829112.679694:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787829112.679694:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787829112.679696:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787829112.679697:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787829112.679698:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829112.679699:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829112.679700:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787829112.679702:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2d2c0 x1874670646797568/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829137 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787829112.679707:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829112.679708:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787829112.679710:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829112.679711:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829112.679712:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829112.679713:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787829112.679715:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787829112.679716:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829112.679718:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787829112.679719:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787829112.679721:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2d2c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646797568:192.168.203.143@tcp:8:kworker.0 00000100:00000001:1.0:1787829112.679723:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829112.679725:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829112.679725:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787829112.679726:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787829112.679728:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787829112.679729:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787829112.679729:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787829112.679731:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787829112.679732:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787829112.679734:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787829112.679735:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829112.679736:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829112.679737:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787829112.679737:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787829112.679740:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787829112.679741:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787829112.679742:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787829112.679743:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787829112.679744:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787829112.679745:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787829112.679746:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829112.679747:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829112.679748:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787829112.679749:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787829112.679750:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787829112.679750:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787829112.679751:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787829112.679752:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787829112.679753:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787829112.679754:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787829112.679756:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829112.679757:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829112.679761:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787829112.679763:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829112.679765:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787829112.679766:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829113.574709:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829113.574719:0:815:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1787829113.574721:0:815:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1787829113.574724:0:815:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829113.574729:0:815:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1787829113.574731:0:815:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829113.574733:0:815:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829113.574735:0:815:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829113.574799:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829113.574803:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829113.574806:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829114.598750:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829114.598758:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829114.598826:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829114.598831:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829114.598835:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829115.622684:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829115.622690:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829115.622748:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829115.622753:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829115.622756:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829116.646690:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829116.646695:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829116.646754:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829116.646759:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829116.646762:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829117.670699:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829117.670706:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829117.670786:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829117.670789:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829117.670792:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787829117.798696:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829117.798704:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1787829117.798706:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787829117.798709:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787829117.798713:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787829117.798720:0:40798:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 3918 00000100:00080000:2.0:1787829117.798729:0:40798:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787829117.798731:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829117.798733:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787829117.798736:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787829117.798738:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787829117.798740:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:2.0:1787829117.798742:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:2.0:1787829117.798743:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829117.798744:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:2.0:1787829117.798748:0:40798:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787829117.798750:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829117.798752:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787829117.798755:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787829117.798757:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829117.798758:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:2.0:1787829117.798760:0:40798:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829117.798769:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787829117.798770:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829117.798773:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787829117.798775:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787829117.798779:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829117.798788:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787829117.798796:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2fc00 x1874670646797696/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787829117.798804:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787829117.798807:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829117.798821:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787829117.798823:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829117.798826:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1787829117.798830:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787829117.798832:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787829117.798834:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787829117.798835:0:40798:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3918 00000100:00080000:2.0:1787829117.798838:0:40798:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787829117.798839:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829117.798840:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787829117.798842:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787829117.798843:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787829117.798845:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000020:00000001:2.0:1787829117.798846:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:2.0:1787829117.798848:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829117.798849:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:2.0:1787829117.798851:0:40798:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787829117.798853:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829117.798855:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787829117.798859:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787829117.798861:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829117.798862:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:1.0:1787829117.798864:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000008:00100000:2.0:1787829117.798866:0:40798:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1787829117.798870:0:40798:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829117.798871:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1787829117.798872:0:40798:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829117.798874:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787829117.798877:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 02000000:00000001:2.0:1787829117.798877:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:1.0:1787829117.798881:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829117.798881:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829117.798884:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1787829117.798887:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787829117.798887:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787829117.798889:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829117.798891:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787829117.798894:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829117.798895:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829117.798905:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646797696:192.168.203.143@tcp:250:kworker.0 00000100:00100000:2.0:1787829117.798905:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2c3c0 x1874670646797824/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787829117.798914:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787829117.798919:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:2.0:1787829117.798919:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787829117.798922:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829117.798923:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829117.798926:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829117.798927:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00080000:2.0:1787829117.798931:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 02000000:00000001:1.0:1787829117.798933:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787829117.798934:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1787829117.798939:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787829117.798940:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787829117.798942:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787829117.798944:0:40798:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3918 00000100:00080000:2.0:1787829117.798946:0:40798:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787829117.798948:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787829117.798949:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787829117.798949:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787829117.798951:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:1.0:1787829117.798952:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417883648 : -126234291667968 : ffff8d30c9e2fc00) 00000020:00000001:2.0:1787829117.798954:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787829117.798957:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:1.0:1787829117.798959:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000020:00000001:2.0:1787829117.798961:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:2.0:1787829117.798964:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829117.798965:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:2.0:1787829117.798968:0:40798:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787829117.798970:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829117.798972:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787829117.798975:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787829117.798978:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829117.798981:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1787829117.798983:0:40798:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1787829117.798986:0:40798:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829117.798987:0:40798:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:1.0:1787829117.798992:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 02000000:00000001:2.0:1787829117.798992:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:1.0:1787829117.799002:0:2390:0:(events.c:34:request_out_callback()) Process entered 02000000:00000001:2.0:1787829117.799005:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829117.799006:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787829117.799010:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:1.0:1787829117.799011:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829117.799015:0:2390:0:(events.c:68:request_out_callback()) Process leaving 02000000:00000001:2.0:1787829117.799015:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787829117.799020:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829117.799024:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829117.799028:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829117.799031:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4298592000 00000100:00100000:2.0:1787829117.799036:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2de00 x1874670646797952/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787829117.799043:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787829117.799045:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 00000100:00000001:2.0:1787829117.799045:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829117.799048:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787829117.799052:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1787829117.799054:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4298592000 00000100:00080000:2.0:1787829117.799055:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1787829117.799057:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787829117.799059:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787829117.799063:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 00000100:00000001:2.0:1787829117.799064:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787829117.799068:0:40798:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3918 00000100:00080000:2.0:1787829117.799071:0:40798:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787829117.799072:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829117.799073:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787829117.799075:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787829117.799077:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:1.0:1787829117.799079:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000020:00000001:2.0:1787829117.799079:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000020:00000001:2.0:1787829117.799085:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:1.0:1787829117.799086:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829117.799087:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829117.799090:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:1.0:1787829117.799091:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00080000:2.0:1787829117.799095:0:40798:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787829117.799099:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829117.799100:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829117/real 1787829117] req@ffff8d30c9e2fc00 x1874670646797696/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829142 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1787829117.799101:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787829117.799105:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787829117.799109:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829117.799113:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1787829117.799116:0:40798:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000400:00000001:1.0:1787829117.799120:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000008:00000001:2.0:1787829117.799120:0:40798:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829117.799124:0:40798:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:1.0:1787829117.799125:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 02000000:00000001:2.0:1787829117.799128:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:1.0:1787829117.799130:0:2390:0:(events.c:78:reply_in_callback()) Process entered 02000000:00000001:2.0:1787829117.799131:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829117.799134:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:1.0:1787829117.799135:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 02000000:00000001:2.0:1787829117.799136:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:1.0:1787829117.799137:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:2.0:1787829117.799139:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829117.799141:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829117.799143:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829117.799144:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829117.799149:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2fc00 x1874670646797696/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829142 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829117.799152:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2c780 x1874670646798080/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787829117.799169:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829117.799172:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787829117.799174:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787829117.799176:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829117.799181:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829117.799181:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829117.799186:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829117.799188:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829117.799191:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787829117.799192:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787829117.799195:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829117.799198:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787829117.799200:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787829117.799204:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646797696:192.168.203.143@tcp:250:kworker.0 00000100:00000001:1.0:1787829117.799208:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829117.799210:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829117.799212:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787829117.799214:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787829117.799216:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787829117.799218:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787829117.799220:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787829117.799223:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787829117.799224:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787829117.799227:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787829117.799228:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829117.799229:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829117.799235:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787829117.799236:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829117.799238:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787829117.799239:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1787829117.799240:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787829117.799241:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787829117.799242:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829117.799244:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787829117.799245:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829117.799246:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829117.799249:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646797824:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787829117.799251:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787829117.799252:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787829117.799253:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829117.799254:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787829117.799256:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829117.799260:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787829117.799261:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417869248 : -126234291682368 : ffff8d30c9e2c3c0) 00000100:00000001:1.0:1787829117.799265:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1787829117.799275:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787829117.799292:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787829117.799294:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829117.799295:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829117.799296:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787829117.799299:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829117.799303:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298590742 00000100:00000001:1.0:1787829117.799305:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298592000 00000100:00000001:1.0:1787829117.799307:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298590742 00000100:00000001:1.0:1787829117.799308:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298592000 00000100:00000001:1.0:1787829117.799310:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829117.799311:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829117.799313:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787829117.799316:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829117/real 1787829117] req@ffff8d30c9e2c3c0 x1874670646797824/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829142 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787829117.799322:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787829117.799323:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787829117.799325:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787829117.799326:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787829117.799327:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829117.799328:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829117.799329:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787829117.799331:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2c3c0 x1874670646797824/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829142 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787829117.799336:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829117.799338:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787829117.799339:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829117.799340:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829117.799341:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829117.799342:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787829117.799343:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787829117.799345:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829117.799347:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787829117.799348:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787829117.799350:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646797824:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787829117.799353:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787829117.799354:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787829117.799355:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829117.799356:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787829117.799357:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829117.799359:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829117.799361:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646797952:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787829117.799363:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787829117.799364:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787829117.799366:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829117.799367:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787829117.799369:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829117.799372:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787829117.799373:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417875968 : -126234291675648 : ffff8d30c9e2de00) 00000100:00000001:1.0:1787829117.799375:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1787829117.799383:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787829117.799386:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787829117.799388:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829117.799389:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829117.799390:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787829117.799391:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829117.799393:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298590742 00000100:00000001:1.0:1787829117.799395:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298592000 00000100:00000001:1.0:1787829117.799396:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298590742 00000100:00000001:1.0:1787829117.799398:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298592000 00000100:00000001:1.0:1787829117.799399:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829117.799400:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829117.799401:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787829117.799403:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829117/real 1787829117] req@ffff8d30c9e2de00 x1874670646797952/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829142 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787829117.799409:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787829117.799409:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787829117.799411:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787829117.799412:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787829117.799413:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829117.799414:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829117.799415:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787829117.799417:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2de00 x1874670646797952/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829142 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787829117.799426:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829117.799427:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787829117.799429:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829117.799430:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829117.799431:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829117.799431:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787829117.799432:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787829117.799434:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829117.799436:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787829117.799437:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787829117.799439:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646797952:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787829117.799441:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787829117.799442:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787829117.799443:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829117.799444:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787829117.799445:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829117.799446:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829117.799448:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646798080:192.168.203.143@tcp:8:kworker.0 00000100:00000001:1.0:1787829117.799449:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787829117.799451:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787829117.799451:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829117.799453:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787829117.799454:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829117.799457:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787829117.799458:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417870208 : -126234291681408 : ffff8d30c9e2c780) 00000100:00000001:1.0:1787829117.799460:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1787829117.799466:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787829117.799468:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787829117.799469:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829117.799470:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829117.799471:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787829117.799473:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829117.799475:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298590755 00000100:00000001:1.0:1787829117.799476:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298592000 00000100:00000001:1.0:1787829117.799478:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298590755 00000100:00000001:1.0:1787829117.799479:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298592000 00000100:00000001:1.0:1787829117.799481:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829117.799482:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829117.799483:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787829117.799485:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829117/real 1787829117] req@ffff8d30c9e2c780 x1874670646798080/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829142 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787829117.799490:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787829117.799491:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787829117.799492:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787829117.799493:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787829117.799493:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829117.799494:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829117.799495:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787829117.799497:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2c780 x1874670646798080/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829142 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787829117.799502:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829117.799503:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787829117.799505:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829117.799506:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829117.799507:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829117.799507:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787829117.799508:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787829117.799510:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829117.799512:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787829117.799513:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787829117.799515:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646798080:192.168.203.143@tcp:8:kworker.0 00000100:00000001:1.0:1787829117.799517:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829117.799518:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829117.799519:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787829117.799520:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787829117.799521:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787829117.799522:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787829117.799523:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787829117.799524:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787829117.799525:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787829117.799527:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787829117.799528:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829117.799529:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829117.799529:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787829117.799530:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787829117.799532:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787829117.799532:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787829117.799533:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787829117.799534:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787829117.799535:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787829117.799539:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787829117.799539:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829117.799540:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829117.799541:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787829117.799542:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787829117.799543:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787829117.799544:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787829117.799545:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787829117.799545:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787829117.799546:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787829117.799547:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787829117.799548:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829117.799549:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829117.799551:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787829117.799552:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829117.799553:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787829117.799554:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829118.694705:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829118.694711:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829118.694740:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829118.694745:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829118.694749:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829119.718671:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829119.718676:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829119.718736:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829119.718740:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829119.718743:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829120.742717:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829120.742724:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829120.742791:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829120.742795:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829120.742798:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829121.766662:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829121.766667:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829121.766729:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829121.766734:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829121.766738:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829122.790684:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829122.790690:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829122.790739:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829122.790742:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829122.790745:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787829122.918712:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829122.918724:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1787829122.918730:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787829122.918734:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787829122.918738:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787829122.918747:0:40798:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 3923 00000100:00080000:0.0:1787829122.918758:0:40798:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787829122.918761:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829122.918763:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787829122.918767:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787829122.918769:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787829122.918772:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:0.0:1787829122.918774:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:0.0:1787829122.918776:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829122.918777:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:0.0:1787829122.918781:0:40798:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787829122.918782:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829122.918785:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787829122.918789:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787829122.918791:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829122.918793:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1787829122.918796:0:40798:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829122.918809:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787829122.918811:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829122.918814:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787829122.918817:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787829122.918821:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829122.918836:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787829122.918845:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d26940 x1874670646798208/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787829122.918853:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787829122.918857:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829122.918871:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787829122.918873:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829122.918876:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1787829122.918877:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787829122.918879:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787829122.918881:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787829122.918882:0:40798:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3923 00000100:00080000:0.0:1787829122.918884:0:40798:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787829122.918885:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829122.918886:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787829122.918889:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787829122.918890:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787829122.918891:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000020:00000001:0.0:1787829122.918893:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:0.0:1787829122.918895:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829122.918896:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:0.0:1787829122.918899:0:40798:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787829122.918901:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829122.918904:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787829122.918909:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787829122.918911:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829122.918913:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:1.0:1787829122.918916:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000008:00100000:0.0:1787829122.918918:0:40798:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1787829122.918923:0:40798:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829122.918925:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787829122.918926:0:40798:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829122.918930:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 02000000:00000001:0.0:1787829122.918932:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:1.0:1787829122.918934:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 02000000:00000001:0.0:1787829122.918935:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829122.918938:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829122.918939:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787829122.918943:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787829122.918946:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829122.918947:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:0.0:1787829122.918953:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829122.918953:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829122.918964:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787829122.918967:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d25680 x1874670646798336/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787829122.918976:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d26940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646798208:192.168.203.143@tcp:250:kworker.0 00000100:00000001:0.0:1787829122.918986:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787829122.918987:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:0.0:1787829122.918991:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829122.918994:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:0.0:1787829122.918996:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829122.918999:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787829122.919000:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829122.919010:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 02000000:00000001:1.0:1787829122.919011:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:0.0:1787829122.919021:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787829122.919023:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787829122.919027:0:40798:0:(import.c:485:import_select_connection()) Process entered 02000000:00000001:1.0:1787829122.919028:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787829122.919037:0:40798:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3923 00000100:00080000:0.0:1787829122.919042:0:40798:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787829122.919043:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829122.919044:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787829122.919046:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787829122.919047:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787829122.919049:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:1.0:1787829122.919049:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787829122.919052:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601346880 : -126234108204736 : ffff8d30d4d26940) 00000020:00000001:0.0:1787829122.919053:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:0.0:1787829122.919062:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787829122.919063:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:0.0:1787829122.919068:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:0.0:1787829122.919072:0:40798:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787829122.919074:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829122.919076:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787829122.919078:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787829122.919080:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829122.919083:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1787829122.919084:0:40798:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1787829122.919085:0:40798:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829122.919087:0:40798:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829122.919091:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787829122.919092:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829122.919093:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787829122.919094:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787829122.919096:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829122.919100:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:1.0:1787829122.919103:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00100000:0.0:1787829122.919110:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d252c0 x1874670646798464/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787829122.919115:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1787829122.919121:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787829122.919125:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829122.919126:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787829122.919134:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829122.919134:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829122.919149:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00080000:0.0:1787829122.919150:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829122.919156:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1787829122.919159:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:1.0:1787829122.919160:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787829122.919168:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787829122.919170:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4298597120 00000100:00000001:0.0:1787829122.919172:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787829122.919179:0:40798:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3923 00000100:00080000:0.0:1787829122.919183:0:40798:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1787829122.919184:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1787829122.919188:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829122.919191:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787829122.919193:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:1.0:1787829122.919193:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4298597120 00000020:00000001:0.0:1787829122.919197:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787829122.919199:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000020:00000001:0.0:1787829122.919201:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:0.0:1787829122.919203:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829122.919204:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:0.0:1787829122.919206:0:40798:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787829122.919208:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829122.919208:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 02000000:00000001:0.0:1787829122.919213:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787829122.919218:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829122.919220:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787829122.919222:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829122.919224:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829122.919226:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:1.0:1787829122.919228:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000008:00100000:0.0:1787829122.919230:0:40798:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1787829122.919233:0:40798:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829122.919237:0:40798:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829122.919238:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829122/real 1787829122] req@ffff8d30d4d26940 x1874670646798208/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829147 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:0.0:1787829122.919259:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787829122.919264:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829122.919272:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787829122.919275:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787829122.919279:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829122.919289:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:1.0:1787829122.919295:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00100000:0.0:1787829122.919297:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d27c00 x1874670646798592/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000001:1.0:1787829122.919298:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787829122.919305:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787829122.919310:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787829122.919312:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787829122.919314:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829122.919316:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829122.919318:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829122.919319:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829122.919322:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787829122.919329:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d26940 x1874670646798208/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829147 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787829122.919336:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829122.919339:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787829122.919342:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829122.919343:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829122.919344:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829122.919346:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787829122.919348:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787829122.919351:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829122.919353:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787829122.919355:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787829122.919358:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d26940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646798208:192.168.203.143@tcp:250:kworker.0 00000100:00000001:1.0:1787829122.919362:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829122.919364:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829122.919365:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787829122.919367:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787829122.919370:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787829122.919372:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787829122.919374:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787829122.919377:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787829122.919378:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787829122.919381:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787829122.919382:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829122.919384:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829122.919390:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787829122.919392:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829122.919393:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787829122.919395:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1787829122.919396:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787829122.919397:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787829122.919398:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829122.919400:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787829122.919402:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829122.919403:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829122.919405:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d25680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646798336:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787829122.919407:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787829122.919409:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787829122.919410:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829122.919411:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787829122.919414:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829122.919418:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787829122.919418:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601342080 : -126234108209536 : ffff8d30d4d25680) 00000100:00000001:1.0:1787829122.919422:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1787829122.919432:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787829122.919435:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787829122.919437:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829122.919438:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829122.919439:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787829122.919441:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829122.919444:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298595862 00000100:00000001:1.0:1787829122.919446:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298597120 00000100:00000001:1.0:1787829122.919447:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298595862 00000100:00000001:1.0:1787829122.919449:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298597120 00000100:00000001:1.0:1787829122.919450:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829122.919452:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829122.919453:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787829122.919455:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829122/real 1787829122] req@ffff8d30d4d25680 x1874670646798336/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829147 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787829122.919465:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787829122.919466:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787829122.919467:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787829122.919468:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787829122.919469:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829122.919470:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829122.919471:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787829122.919473:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d25680 x1874670646798336/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829147 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787829122.919478:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829122.919479:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787829122.919481:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829122.919482:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829122.919483:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829122.919484:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787829122.919485:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787829122.919487:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829122.919489:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787829122.919490:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787829122.919492:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d25680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646798336:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787829122.919494:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787829122.919495:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787829122.919496:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829122.919498:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787829122.919499:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829122.919500:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829122.919501:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d252c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646798464:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787829122.919504:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787829122.919505:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787829122.919507:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829122.919508:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787829122.919509:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829122.919512:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787829122.919513:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601341120 : -126234108210496 : ffff8d30d4d252c0) 00000100:00000001:1.0:1787829122.919516:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1787829122.919521:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787829122.919524:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787829122.919525:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829122.919526:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829122.919527:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787829122.919529:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829122.919531:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298595862 00000100:00000001:1.0:1787829122.919532:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298597120 00000100:00000001:1.0:1787829122.919534:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298595862 00000100:00000001:1.0:1787829122.919535:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298597120 00000100:00000001:1.0:1787829122.919537:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829122.919538:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829122.919539:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787829122.919541:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829122/real 1787829122] req@ffff8d30d4d252c0 x1874670646798464/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829147 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787829122.919546:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787829122.919546:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787829122.919548:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787829122.919548:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787829122.919549:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829122.919550:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829122.919551:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787829122.919553:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d252c0 x1874670646798464/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829147 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787829122.919557:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829122.919559:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787829122.919560:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829122.919561:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829122.919562:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829122.919563:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787829122.919564:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787829122.919566:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829122.919568:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787829122.919569:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787829122.919571:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d252c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646798464:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787829122.919573:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787829122.919574:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787829122.919575:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829122.919576:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787829122.919577:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829122.919578:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829122.919580:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d27c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646798592:192.168.203.143@tcp:8:kworker.0 00000100:00000001:1.0:1787829122.919582:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787829122.919583:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787829122.919584:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829122.919585:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787829122.919586:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829122.919589:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787829122.919590:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601351680 : -126234108199936 : ffff8d30d4d27c00) 00000100:00000001:1.0:1787829122.919592:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1787829122.919597:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787829122.919626:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787829122.919628:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829122.919629:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829122.919630:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787829122.919632:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829122.919635:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298595876 00000100:00000001:1.0:1787829122.919637:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298597120 00000100:00000001:1.0:1787829122.919638:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298595876 00000100:00000001:1.0:1787829122.919640:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298597120 00000100:00000001:1.0:1787829122.919641:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829122.919643:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829122.919644:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787829122.919646:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829122/real 1787829122] req@ffff8d30d4d27c00 x1874670646798592/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829147 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787829122.919651:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787829122.919652:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787829122.919653:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787829122.919654:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787829122.919655:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829122.919657:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829122.919658:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787829122.919660:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d27c00 x1874670646798592/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829147 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787829122.919664:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829122.919665:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787829122.919667:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829122.919668:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829122.919669:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829122.919670:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787829122.919671:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787829122.919673:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829122.919675:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787829122.919676:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787829122.919678:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d27c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646798592:192.168.203.143@tcp:8:kworker.0 00000100:00000001:1.0:1787829122.919680:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829122.919681:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829122.919682:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787829122.919683:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787829122.919685:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787829122.919686:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787829122.919687:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787829122.919688:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787829122.919689:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787829122.919692:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787829122.919693:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829122.919694:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829122.919695:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787829122.919696:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787829122.919697:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787829122.919698:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787829122.919699:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787829122.919701:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787829122.919702:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787829122.919704:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787829122.919705:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829122.919707:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829122.919708:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787829122.919709:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787829122.919711:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787829122.919712:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787829122.919713:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787829122.919715:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787829122.919716:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787829122.919718:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787829122.919720:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829122.919722:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829122.919725:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787829122.919726:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829122.919728:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787829122.919730:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829123.814694:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829123.814700:0:815:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1787829123.814702:0:815:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1787829123.814704:0:815:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829123.814708:0:815:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1787829123.814710:0:815:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829123.814711:0:815:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829123.814725:0:815:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829123.814765:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829123.814769:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829123.814772:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829124.838731:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829124.838739:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829124.838806:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829124.838813:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829124.838818:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829125.862698:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829125.862704:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829125.862803:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829125.862806:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829125.862809:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829126.886698:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829126.886703:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829126.886753:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829126.886757:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829126.886760:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829127.910707:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829127.910714:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829127.910772:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829127.910777:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829127.910780:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787829128.038717:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829128.038729:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1787829128.038733:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787829128.038737:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787829128.038742:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787829128.038752:0:468:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 3928 00000100:00080000:3.0:1787829128.038762:0:468:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787829128.038766:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829128.038768:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:3.0:1787829128.038771:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1787829128.038773:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1787829128.038776:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:3.0:1787829128.038777:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:3.0:1787829128.038779:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829128.038780:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:3.0:1787829128.038783:0:468:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:3.0:1787829128.038785:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829128.038787:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787829128.038790:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829128.038792:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829128.038794:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:3.0:1787829128.038796:0:468:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829128.038805:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787829128.038806:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829128.038809:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787829128.038811:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787829128.038815:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829128.038824:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829128.038831:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c70143c0 x1874670646798720/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787829128.038839:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787829128.038843:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829128.038854:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787829128.038856:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829128.038859:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1787829128.038860:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787829128.038861:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787829128.038863:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787829128.038865:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3928 00000100:00080000:3.0:1787829128.038867:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787829128.038868:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829128.038870:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:3.0:1787829128.038872:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1787829128.038874:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1787829128.038876:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000020:00000001:3.0:1787829128.038879:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:3.0:1787829128.038882:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787829128.038883:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1787829128.038884:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:1.0:1787829128.038889:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00080000:3.0:1787829128.038892:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:1.0:1787829128.038893:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787829128.038898:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829128.038900:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 02000000:00000001:3.0:1787829128.038901:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1787829128.038903:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829128.038908:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787829128.038910:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829128.038910:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829128.038917:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829128.038917:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829128.038924:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00100000:1.0:1787829128.038926:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c70143c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646798720:192.168.203.143@tcp:250:kworker.0 00000008:00100000:3.0:1787829128.038930:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00000001:1.0:1787829128.038931:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787829128.038934:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000008:00000001:3.0:1787829128.038936:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829128.038937:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829128.038941:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829128.038942:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787829128.038946:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829128.038949:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787829128.038952:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829128.038955:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787829128.038957:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787829128.038959:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829128.038961:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787829128.038964:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369536448 : -126234340015168 : ffff8d30c70143c0) 00000100:00000001:3.0:1787829128.038966:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829128.038967:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00100000:3.0:1787829128.038977:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c7015680 x1874670646798848/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787829128.038985:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787829128.038986:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829128.038989:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787829128.038992:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829128.038995:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1787829128.038996:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787829128.038997:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787829128.038999:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787829128.039000:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3928 00000100:00080000:3.0:1787829128.039005:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000800:00000100:1.0:1787829128.039006:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:3.0:1787829128.039008:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829128.039011:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:3.0:1787829128.039014:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1787829128.039015:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1787829128.039017:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:1.0:1787829128.039019:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000020:00000001:3.0:1787829128.039020:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:3.0:1787829128.039025:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787829128.039027:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829128.039027:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:1.0:1787829128.039032:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787829128.039036:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:1.0:1787829128.039037:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1787829128.039042:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829128.039045:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829128.039045:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1787829128.039052:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4298602240 00000100:00000001:1.0:1787829128.039060:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 02000000:00000001:3.0:1787829128.039060:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829128.039065:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829128.039067:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:1.0:1787829128.039069:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4298602240 00000008:00100000:3.0:1787829128.039070:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1787829128.039074:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829128.039075:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 00000100:00000001:3.0:1787829128.039076:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829128.039079:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829128.039083:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829128.039084:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:1.0:1787829128.039085:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 02000000:00000001:3.0:1787829128.039089:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829128.039093:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829128/real 1787829128] req@ffff8d30c70143c0 x1874670646798720/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829153 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829128.039095:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787829128.039098:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787829128.039102:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:1.0:1787829128.039108:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787829128.039110:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829128.039111:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829128.039113:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787829128.039117:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787829128.039119:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829128.039122:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829128.039124:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829128.039124:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c70152c0 x1874670646798976/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787829128.039132:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c70143c0 x1874670646798720/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829153 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829128.039149:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787829128.039150:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829128.039155:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829128.039156:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829128.039157:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829128.039161:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829128.039162:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829128.039163:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829128.039164:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787829128.039167:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787829128.039170:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829128.039170:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829128.039176:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787829128.039180:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00080000:3.0:1787829128.039180:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1787829128.039185:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00100000:1.0:1787829128.039186:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c70143c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646798720:192.168.203.143@tcp:250:kworker.0 00000100:00080000:3.0:1787829128.039190:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787829128.039191:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829128.039195:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829128.039197:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 00000100:00000001:3.0:1787829128.039197:0:468:0:(import.c:485:import_select_connection()) Process entered 02000000:00000001:1.0:1787829128.039199:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787829128.039202:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00080000:3.0:1787829128.039204:0:468:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3928 00000020:00000001:1.0:1787829128.039205:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787829128.039223:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787829128.039229:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 00000100:00080000:3.0:1787829128.039229:0:468:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 02000000:00000001:1.0:1787829128.039231:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787829128.039235:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829128.039235:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787829128.039238:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829128.039240:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:1.0:1787829128.039241:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:3.0:1787829128.039246:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1787829128.039249:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:1.0:1787829128.039250:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000020:00000001:3.0:1787829128.039251:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:1.0:1787829128.039252:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829128.039256:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787829128.039258:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000020:00000001:3.0:1787829128.039258:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:1.0:1787829128.039260:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787829128.039262:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787829128.039266:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829128.039266:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829128.039270:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 02000000:00000001:1.0:1787829128.039271:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787829128.039273:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829128.039277:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787829128.039280:0:468:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00100000:1.0:1787829128.039282:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c7015680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646798848:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787829128.039289:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:3.0:1787829128.039289:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829128.039292:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787829128.039296:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829128.039296:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1787829128.039300:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787829128.039303:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829128.039303:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829128.039308:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829128.039309:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787829128.039311:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369541248 : -126234340010368 : ffff8d30c7015680) 00000100:00000001:3.0:1787829128.039313:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:1.0:1787829128.039316:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000008:00100000:3.0:1787829128.039319:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1787829128.039324:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829128.039326:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:1.0:1787829128.039330:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787829128.039335:0:2390:0:(events.c:34:request_out_callback()) Process entered 02000000:00000001:3.0:1787829128.039335:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787829128.039339:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829128.039340:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829128.039342:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829128.039345:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1787829128.039345:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:1.0:1787829128.039347:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829128.039351:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298600982 00000100:00000001:1.0:1787829128.039354:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298602240 02000000:00000001:3.0:1787829128.039357:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:1.0:1787829128.039359:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298600982 02000000:00000001:3.0:1787829128.039361:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829128.039364:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298602240 00000100:00000001:1.0:1787829128.039368:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829128.039369:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829128.039371:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829128.039375:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1787829128.039379:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c7014b40 x1874670646799104/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787829128.039383:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829128/real 1787829128] req@ffff8d30c7015680 x1874670646798848/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829153 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829128.039399:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000400:00000001:1.0:1787829128.039404:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000001:3.0:1787829128.039405:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:1.0:1787829128.039408:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829128.039412:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829128.039413:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787829128.039417:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787829128.039419:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829128.039421:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829128.039422:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787829128.039426:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c7015680 x1874670646798848/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829153 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787829128.039431:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829128.039433:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787829128.039435:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829128.039436:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829128.039437:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829128.039438:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787829128.039440:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787829128.039443:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829128.039445:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787829128.039447:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787829128.039449:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c7015680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646798848:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787829128.039452:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787829128.039452:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787829128.039453:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829128.039455:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787829128.039456:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829128.039458:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829128.039459:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c70152c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646798976:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787829128.039462:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787829128.039463:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787829128.039464:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829128.039465:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787829128.039468:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829128.039471:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787829128.039472:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369540288 : -126234340011328 : ffff8d30c70152c0) 00000100:00000001:1.0:1787829128.039475:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1787829128.039484:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787829128.039487:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787829128.039489:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829128.039490:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829128.039491:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787829128.039493:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829128.039495:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298600982 00000100:00000001:1.0:1787829128.039497:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298602240 00000100:00000001:1.0:1787829128.039498:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298600982 00000100:00000001:1.0:1787829128.039500:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298602240 00000100:00000001:1.0:1787829128.039501:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829128.039503:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829128.039504:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787829128.039506:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829128/real 1787829128] req@ffff8d30c70152c0 x1874670646798976/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829153 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787829128.039511:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787829128.039512:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787829128.039521:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787829128.039522:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787829128.039523:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829128.039524:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829128.039525:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787829128.039528:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c70152c0 x1874670646798976/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829153 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787829128.039532:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829128.039534:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787829128.039536:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829128.039536:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829128.039537:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829128.039538:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787829128.039539:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787829128.039541:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829128.039543:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787829128.039545:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787829128.039547:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c70152c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646798976:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787829128.039549:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829128.039551:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829128.039551:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787829128.039552:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787829128.039554:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787829128.039555:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787829128.039555:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787829128.039557:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787829128.039558:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787829128.039559:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787829128.039560:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829128.039561:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829128.039562:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787829128.039563:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787829128.039565:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787829128.039565:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787829128.039566:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787829128.039567:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787829128.039568:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787829128.039569:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787829128.039570:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829128.039571:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829128.039573:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787829128.039574:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829128.039575:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787829128.039576:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1787829128.039577:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787829128.039577:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787829128.039578:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829128.039586:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787829128.039587:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829128.039589:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829128.039591:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c7014b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646799104:192.168.203.143@tcp:8:kworker.0 00000100:00000001:1.0:1787829128.039593:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787829128.039594:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787829128.039595:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829128.039597:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787829128.039598:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829128.039643:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787829128.039646:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369538368 : -126234340013248 : ffff8d30c7014b40) 00000100:00000001:1.0:1787829128.039650:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1787829128.039663:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787829128.039666:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787829128.039668:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829128.039669:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829128.039670:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787829128.039672:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829128.039675:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298600996 00000100:00000001:1.0:1787829128.039676:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298602240 00000100:00000001:1.0:1787829128.039689:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298600996 00000100:00000001:1.0:1787829128.039691:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298602240 00000100:00000001:1.0:1787829128.039692:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829128.039693:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829128.039695:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787829128.039697:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829128/real 1787829128] req@ffff8d30c7014b40 x1874670646799104/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829153 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787829128.039702:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787829128.039703:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787829128.039705:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787829128.039706:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787829128.039707:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829128.039708:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829128.039709:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787829128.039711:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c7014b40 x1874670646799104/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829153 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787829128.039716:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829128.039717:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787829128.039719:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829128.039720:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829128.039721:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829128.039722:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787829128.039723:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787829128.039726:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829128.039728:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787829128.039729:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787829128.039731:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c7014b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646799104:192.168.203.143@tcp:8:kworker.0 00000100:00000001:1.0:1787829128.039733:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829128.039734:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829128.039735:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787829128.039736:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787829128.039738:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787829128.039739:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787829128.039739:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787829128.039741:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787829128.039742:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787829128.039754:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787829128.039755:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829128.039757:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829128.039760:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787829128.039761:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829128.039763:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787829128.039765:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829128.934681:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829128.934687:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829128.934731:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829128.934734:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829128.934737:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829129.958714:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829129.958719:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829129.958770:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829129.958773:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829129.958775:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829130.982733:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829130.982740:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829130.982807:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829130.982811:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829130.982814:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829132.006803:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829132.006813:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829132.006886:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829132.006890:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829132.006893:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829133.030728:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829133.030736:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829133.030804:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829133.030926:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829133.030930:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787829133.158769:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829133.158783:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1787829133.158786:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1787829133.158791:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787829133.158796:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1787829133.158808:0:468:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 3933 00000100:00080000:1.0:1787829133.158823:0:468:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1787829133.158827:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787829133.158828:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:1.0:1787829133.158832:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:1.0:1787829133.158835:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:1.0:1787829133.158838:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:1.0:1787829133.158839:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:1.0:1787829133.158841:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787829133.158842:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:1.0:1787829133.158847:0:468:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:1.0:1787829133.158849:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829133.158852:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1787829133.158855:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829133.158858:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829133.158860:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:1.0:1787829133.158862:0:468:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829133.158876:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1787829133.158878:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829133.158881:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1787829133.158884:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1787829133.158889:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829133.158904:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829133.158916:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d13ef0c0 x1874670646799232/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787829133.158929:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787829133.158935:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829133.158952:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1787829133.158955:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829133.158958:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1787829133.158959:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1787829133.158960:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787829133.158963:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1787829133.158965:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3933 00000100:00080000:1.0:1787829133.158969:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1787829133.158970:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787829133.158972:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:1.0:1787829133.158973:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:1.0:1787829133.158974:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:1.0:1787829133.158976:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000020:00000001:1.0:1787829133.158977:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:1.0:1787829133.158979:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787829133.158980:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:1.0:1787829133.158982:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:1.0:1787829133.158984:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829133.158985:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1787829133.158991:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829133.158993:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829133.158994:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:1.0:1787829133.158998:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1787829133.159001:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829133.159003:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829133.159007:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1787829133.159008:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829133.159010:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1787829133.159011:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1787829133.159013:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829133.159018:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829133.159022:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d13ede00 x1874670646799360/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787829133.159028:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787829133.159029:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829133.159033:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829133.159035:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00080000:1.0:1787829133.159035:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829133.159041:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:1.0:1787829133.159043:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1787829133.159045:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787829133.159046:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1787829133.159048:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1787829133.159053:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3933 00000100:00000001:0.0:1787829133.159054:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00080000:1.0:1787829133.159056:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1787829133.159059:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787829133.159060:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:0.0:1787829133.159061:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000020:00000001:1.0:1787829133.159063:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:1.0:1787829133.159065:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:0.0:1787829133.159066:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829133.159067:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000020:00000001:1.0:1787829133.159074:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:1.0:1787829133.159077:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787829133.159078:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 02000000:00000001:0.0:1787829133.159079:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000100:00080000:1.0:1787829133.159080:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:1.0:1787829133.159088:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829133.159090:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1787829133.159092:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787829133.159093:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829133.159095:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829133.159101:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829133.159102:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:1.0:1787829133.159106:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1787829133.159111:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829133.159116:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787829133.159117:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d13ef0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646799232:192.168.203.143@tcp:250:kworker.0 02000000:00000001:1.0:1787829133.159125:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:0.0:1787829133.159128:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787829133.159129:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829133.159134:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1787829133.159135:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787829133.159136:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787829133.159139:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829133.159143:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829133.159148:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:1.0:1787829133.159148:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829133.159158:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d13eed00 x1874670646799488/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:0.0:1787829133.159160:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829133.159188:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787829133.159191:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829133.159196:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1787829133.159198:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1787829133.159199:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00080000:1.0:1787829133.159201:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1787829133.159204:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839541350592 : -126234168201024 : ffff8d30d13ef0c0) 00000100:00000001:1.0:1787829133.159205:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1787829133.159206:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787829133.159214:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:0.0:1787829133.159217:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00080000:1.0:1787829133.159221:0:468:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3933 00000100:00080000:1.0:1787829133.159227:0:468:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1787829133.159229:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787829133.159230:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:1.0:1787829133.159232:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:1.0:1787829133.159234:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:1.0:1787829133.159236:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000020:00000001:1.0:1787829133.159238:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:1.0:1787829133.159240:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787829133.159241:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:1.0:1787829133.159244:0:468:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:1.0:1787829133.159245:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829133.159246:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1787829133.159249:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829133.159251:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829133.159253:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:1.0:1787829133.159255:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000008:00000001:1.0:1787829133.159256:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829133.159258:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829133.159261:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1787829133.159263:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829133.159271:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000800:00000100:0.0:1787829133.159272:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 02000000:00000001:1.0:1787829133.159274:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1787829133.159279:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829133.159287:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787829133.159288:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829133.159294:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00100000:1.0:1787829133.159297:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d13ef840 x1874670646799616/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787829133.159299:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829133.159308:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787829133.159318:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787829133.159321:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829133.159325:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829133.159326:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829133.159334:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4298607360 00000100:00000001:0.0:1787829133.159346:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1787829133.159354:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4298607360 00000100:00000001:0.0:1787829133.159359:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1787829133.159363:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787829133.159366:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829133.159369:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787829133.159377:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829133/real 1787829133] req@ffff8d30d13ef0c0 x1874670646799232/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829158 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787829133.159390:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787829133.159392:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787829133.159396:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787829133.159398:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787829133.159400:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829133.159403:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787829133.159405:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787829133.159410:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d13ef0c0 x1874670646799232/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829158 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787829133.159419:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829133.159423:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787829133.159426:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829133.159428:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787829133.159429:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829133.159432:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787829133.159435:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787829133.159439:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829133.159444:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787829133.159447:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787829133.159451:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d13ef0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646799232:192.168.203.143@tcp:250:kworker.0 00000100:00000001:0.0:1787829133.159457:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1787829133.159458:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787829133.159460:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829133.159463:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787829133.159465:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787829133.159468:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787829133.159471:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d13ede00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646799360:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787829133.159475:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787829133.159478:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787829133.159480:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829133.159481:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1787829133.159486:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829133.159492:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787829133.159493:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839541345792 : -126234168205824 : ffff8d30d13ede00) 00000100:00000001:0.0:1787829133.159498:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1787829133.159513:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:0.0:1787829133.159518:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1787829133.159521:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787829133.159522:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829133.159524:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787829133.159527:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829133.159532:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298606102 00000100:00000001:0.0:1787829133.159535:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298607360 00000100:00000001:0.0:1787829133.159538:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298606102 00000100:00000001:0.0:1787829133.159541:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298607360 00000100:00000001:0.0:1787829133.159543:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787829133.159545:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829133.159547:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787829133.159552:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829133/real 1787829133] req@ffff8d30d13ede00 x1874670646799360/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829158 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787829133.159561:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787829133.159563:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787829133.159566:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787829133.159568:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787829133.159569:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829133.159571:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787829133.159572:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787829133.159577:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d13ede00 x1874670646799360/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829158 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787829133.159585:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829133.159587:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787829133.159590:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829133.159591:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787829133.159593:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829133.159595:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787829133.159597:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787829133.159647:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829133.159653:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787829133.159656:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787829133.159660:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d13ede00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646799360:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787829133.159665:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829133.159667:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787829133.159669:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787829133.159672:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787829133.159675:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787829133.159678:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787829133.159680:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787829133.159685:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787829133.159687:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787829133.159690:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787829133.159691:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829133.159693:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787829133.159695:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787829133.159696:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787829133.159698:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787829133.159700:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787829133.159701:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787829133.159703:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787829133.159705:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787829133.159707:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787829133.159709:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829133.159711:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829133.159718:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787829133.159721:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829133.159723:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787829133.159725:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787829133.159726:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1787829133.159728:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787829133.159730:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829133.159732:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787829133.159734:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787829133.159736:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787829133.159740:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d13eed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646799488:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787829133.159744:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787829133.159747:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787829133.159748:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829133.159750:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1787829133.159753:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829133.159759:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787829133.159761:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839541349632 : -126234168201984 : ffff8d30d13eed00) 00000100:00000001:0.0:1787829133.159765:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1787829133.159796:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:0.0:1787829133.159809:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1787829133.159813:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787829133.159814:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829133.159816:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787829133.159819:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829133.159823:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298606103 00000100:00000001:0.0:1787829133.159826:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298607360 00000100:00000001:0.0:1787829133.159829:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298606103 00000100:00000001:0.0:1787829133.159831:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298607360 00000100:00000001:0.0:1787829133.159833:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787829133.159835:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829133.159837:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787829133.159843:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829133/real 1787829133] req@ffff8d30d13eed00 x1874670646799488/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829158 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787829133.159852:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787829133.159854:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787829133.159856:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787829133.159858:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787829133.159860:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829133.159862:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787829133.159863:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787829133.159868:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d13eed00 x1874670646799488/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829158 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787829133.159876:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829133.159878:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787829133.159881:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829133.159883:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787829133.159884:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829133.159886:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787829133.159888:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787829133.159891:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829133.159896:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787829133.159899:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787829133.159903:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d13eed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646799488:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787829133.159908:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1787829133.159910:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787829133.159911:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829133.159914:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787829133.159916:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787829133.159919:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787829133.159922:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d13ef840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646799616:192.168.203.143@tcp:8:kworker.0 00000100:00000001:0.0:1787829133.159926:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787829133.159928:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787829133.159930:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829133.159932:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1787829133.159935:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829133.159940:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787829133.159942:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839541352512 : -126234168199104 : ffff8d30d13ef840) 00000100:00000001:0.0:1787829133.159946:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1787829133.159958:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:0.0:1787829133.159963:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1787829133.159966:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787829133.159967:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829133.159969:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787829133.159972:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829133.159976:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298606116 00000100:00000001:0.0:1787829133.159979:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298607360 00000100:00000001:0.0:1787829133.159981:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298606116 00000100:00000001:0.0:1787829133.159984:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298607360 00000100:00000001:0.0:1787829133.159986:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787829133.159989:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829133.159990:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787829133.159995:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829133/real 1787829133] req@ffff8d30d13ef840 x1874670646799616/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829158 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787829133.160004:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787829133.160006:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787829133.160008:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787829133.160010:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787829133.160012:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829133.160014:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787829133.160015:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787829133.160019:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d13ef840 x1874670646799616/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829158 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787829133.160028:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829133.160030:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787829133.160032:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829133.160034:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787829133.160035:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829133.160037:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787829133.160039:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787829133.160042:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829133.160047:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787829133.160049:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787829133.160053:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d13ef840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646799616:192.168.203.143@tcp:8:kworker.0 00000100:00000001:0.0:1787829133.160057:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829133.160059:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787829133.160060:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787829133.160062:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787829133.160065:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787829133.160067:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787829133.160068:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787829133.160070:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787829133.160072:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787829133.160074:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787829133.160076:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829133.160078:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787829133.160079:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787829133.160081:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787829133.160083:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787829133.160084:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787829133.160085:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787829133.160087:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787829133.160088:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787829133.160090:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787829133.160091:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829133.160093:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829133.160097:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787829133.160098:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829133.160100:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787829133.160102:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829134.054659:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829134.054665:0:815:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1787829134.054667:0:815:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1787829134.054670:0:815:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829134.054673:0:815:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1787829134.054681:0:815:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829134.054682:0:815:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829134.054684:0:815:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829134.054759:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829134.054764:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829134.054768:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829135.078656:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829135.078661:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829135.078714:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829135.078719:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829135.078723:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829136.102664:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829136.102668:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829136.102717:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829136.102720:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829136.102723:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829137.126659:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829137.126664:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829137.126751:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829137.126754:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829137.126757:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829138.150658:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829138.150663:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829138.150706:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829138.150709:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829138.150712:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787829138.278708:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829138.278719:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1787829138.278722:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787829138.278726:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787829138.278731:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787829138.278755:0:468:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 3938 00000100:00080000:0.0:1787829138.278766:0:468:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787829138.278770:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829138.278772:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787829138.278776:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787829138.278779:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787829138.278782:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:0.0:1787829138.278784:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:0.0:1787829138.278787:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829138.278788:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:0.0:1787829138.278793:0:468:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787829138.278796:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829138.278799:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787829138.278803:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787829138.278806:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829138.278809:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1787829138.278811:0:468:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829138.278823:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787829138.278826:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829138.278829:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787829138.278832:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787829138.278836:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829138.278846:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787829138.278856:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d270c0 x1874670646799744/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787829138.278867:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787829138.278872:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829138.278890:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787829138.278894:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829138.278898:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1787829138.278900:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787829138.278902:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787829138.278906:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787829138.278909:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3938 00000100:00000001:1.0:1787829138.278912:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00080000:0.0:1787829138.278914:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1787829138.278917:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787829138.278919:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787829138.278921:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1787829138.278923:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:1.0:1787829138.278924:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787829138.278927:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787829138.278931:0:468:0:(genops.c:792:class_conn2export()) Process entered 02000000:00000001:1.0:1787829138.278933:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000020:00000001:0.0:1787829138.278935:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 02000000:00000001:1.0:1787829138.278936:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000020:00000001:0.0:1787829138.278940:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 02000000:00000001:1.0:1787829138.278940:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787829138.278946:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00100000:1.0:1787829138.278947:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d270c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646799744:192.168.203.143@tcp:250:kworker.0 00000100:00000001:0.0:1787829138.278953:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787829138.278954:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:0.0:1787829138.278958:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 02000000:00000001:1.0:1787829138.278960:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787829138.278963:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829138.278966:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00080000:0.0:1787829138.278968:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 02000000:00000001:1.0:1787829138.278972:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829138.278973:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829138.278978:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787829138.278985:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829138.278986:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 02000000:00000001:0.0:1787829138.278988:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829138.278989:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601348800 : -126234108202816 : ffff8d30d4d270c0) 00000100:00000001:0.0:1787829138.278994:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:1.0:1787829138.278995:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000008:00100000:0.0:1787829138.279002:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1787829138.279007:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829138.279010:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829138.279015:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787829138.279017:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829138.279019:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787829138.279020:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787829138.279022:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829138.279028:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787829138.279035:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d25a40 x1874670646799872/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787829138.279042:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:0.0:1787829138.279049:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787829138.279053:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829138.279055:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1787829138.279058:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829138.279060:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829138.279064:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787829138.279065:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1787829138.279066:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00080000:0.0:1787829138.279072:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:1.0:1787829138.279072:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829138.279076:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:1.0:1787829138.279078:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4298612480 00000100:00080000:0.0:1787829138.279080:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787829138.279085:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:1.0:1787829138.279088:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 00000100:00080000:0.0:1787829138.279090:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3938 00000100:00000001:1.0:1787829138.279096:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4298612480 00000100:00080000:0.0:1787829138.279097:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1787829138.279100:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1787829138.279102:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787829138.279104:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787829138.279106:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:1.0:1787829138.279107:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787829138.279113:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:1.0:1787829138.279114:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000020:00000001:0.0:1787829138.279118:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00100000:1.0:1787829138.279121:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829138/real 1787829138] req@ffff8d30d4d270c0 x1874670646799744/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829163 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:0.0:1787829138.279123:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000020:00000001:0.0:1787829138.279129:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000400:00000001:1.0:1787829138.279149:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000001:0.0:1787829138.279150:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000001:1.0:1787829138.279151:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787829138.279156:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:1.0:1787829138.279157:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787829138.279161:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787829138.279163:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:0.0:1787829138.279166:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:1.0:1787829138.279166:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829138.279169:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829138.279174:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829138.279177:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d270c0 x1874670646799744/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829163 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:0.0:1787829138.279178:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787829138.279184:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829138.279187:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:0.0:1787829138.279188:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829138.279191:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787829138.279193:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:1.0:1787829138.279194:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829138.279197:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000008:00100000:0.0:1787829138.279198:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00000001:1.0:1787829138.279199:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829138.279203:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000008:00000001:0.0:1787829138.279206:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829138.279206:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787829138.279213:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1787829138.279213:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829138.279219:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 02000000:00000001:0.0:1787829138.279222:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:1.0:1787829138.279223:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 02000000:00000001:0.0:1787829138.279227:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829138.279228:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d270c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646799744:192.168.203.143@tcp:250:kworker.0 00000100:00000001:0.0:1787829138.279233:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:1.0:1787829138.279234:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:0.0:1787829138.279236:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:1.0:1787829138.279237:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829138.279240:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787829138.279242:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829138.279243:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787829138.279248:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00000001:0.0:1787829138.279251:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829138.279251:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787829138.279253:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787829138.279258:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787829138.279261:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00100000:0.0:1787829138.279262:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d24f00 x1874670646800000/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787829138.279264:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787829138.279266:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829138.279270:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829138.279278:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787829138.279279:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787829138.279281:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829138.279284:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829138.279285:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787829138.279287:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1787829138.279290:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787829138.279292:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787829138.279294:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829138.279294:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829138.279299:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787829138.279302:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00080000:0.0:1787829138.279303:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 02000000:00000001:1.0:1787829138.279304:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829138.279311:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d25a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646799872:192.168.203.143@tcp:38:kworker.0 00000100:00080000:0.0:1787829138.279312:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1787829138.279316:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:1.0:1787829138.279328:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00080000:0.0:1787829138.279329:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 02000000:00000001:1.0:1787829138.279331:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787829138.279333:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829138.279335:0:468:0:(import.c:485:import_select_connection()) Process entered 02000000:00000001:1.0:1787829138.279336:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787829138.279340:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787829138.279342:0:468:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3938 00000100:00000001:1.0:1787829138.279345:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00080000:0.0:1787829138.279348:0:468:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1787829138.279348:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601343040 : -126234108208576 : ffff8d30d4d25a40) 00000100:00000001:1.0:1787829138.279356:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:0.0:1787829138.279361:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829138.279364:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787829138.279366:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000800:00000100:1.0:1787829138.279369:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000020:00000001:0.0:1787829138.279370:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787829138.279374:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:1.0:1787829138.279376:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000020:00000001:0.0:1787829138.279377:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:1.0:1787829138.279380:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787829138.279391:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787829138.279393:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829138.279394:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:1.0:1787829138.279400:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00080000:0.0:1787829138.279402:0:468:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787829138.279406:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829138.279407:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1787829138.279407:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829138.279411:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787829138.279415:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829138.279416:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298611222 00000100:00000001:0.0:1787829138.279419:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1787829138.279423:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00000001:1.0:1787829138.279424:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298612480 00000008:00000001:0.0:1787829138.279426:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829138.279430:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829138.279431:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298611222 02000000:00000001:0.0:1787829138.279434:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787829138.279437:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829138.279437:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298612480 00000100:00000001:0.0:1787829138.279440:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:1.0:1787829138.279444:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829138.279457:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829138.279458:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:1.0:1787829138.279459:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 02000000:00000001:0.0:1787829138.279462:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829138.279467:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829138/real 1787829138] req@ffff8d30d4d25a40 x1874670646799872/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829163 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787829138.279470:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787829138.279479:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d24000 x1874670646800128/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000001:1.0:1787829138.279480:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787829138.279484:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787829138.279490:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787829138.279491:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787829138.279494:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787829138.279495:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829138.279499:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829138.279499:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829138.279506:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829138.279508:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787829138.279515:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d25a40 x1874670646799872/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829163 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787829138.279524:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829138.279527:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787829138.279530:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829138.279531:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829138.279533:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829138.279535:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787829138.279536:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787829138.279539:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829138.279542:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787829138.279543:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787829138.279546:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d25a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646799872:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787829138.279549:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829138.279550:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829138.279551:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787829138.279552:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787829138.279554:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787829138.279555:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787829138.279556:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787829138.279558:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787829138.279559:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787829138.279561:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787829138.279562:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829138.279563:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829138.279566:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787829138.279567:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829138.279568:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787829138.279569:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1787829138.279570:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787829138.279572:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787829138.279573:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829138.279574:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787829138.279575:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829138.279577:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829138.279579:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d24f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646800000:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787829138.279581:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787829138.279582:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787829138.279583:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829138.279585:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787829138.279587:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829138.279591:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787829138.279592:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601340160 : -126234108211456 : ffff8d30d4d24f00) 00000100:00000001:1.0:1787829138.279594:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1787829138.279622:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787829138.279633:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787829138.279634:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829138.279635:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829138.279637:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787829138.279638:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829138.279641:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298611223 00000100:00000001:1.0:1787829138.279643:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298612480 00000100:00000001:1.0:1787829138.279645:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298611223 00000100:00000001:1.0:1787829138.279646:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298612480 00000100:00000001:1.0:1787829138.279647:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829138.279649:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829138.279650:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787829138.279653:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829138/real 1787829138] req@ffff8d30d4d24f00 x1874670646800000/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829163 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787829138.279659:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787829138.279659:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787829138.279661:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787829138.279662:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787829138.279663:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829138.279664:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829138.279665:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787829138.279668:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d24f00 x1874670646800000/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829163 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787829138.279672:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829138.279673:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787829138.279675:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829138.279676:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829138.279676:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829138.279677:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787829138.279678:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787829138.279680:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829138.279682:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787829138.279683:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787829138.279685:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d24f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646800000:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787829138.279688:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787829138.279689:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787829138.279690:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829138.279691:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787829138.279692:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829138.279694:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829138.279695:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d24000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646800128:192.168.203.143@tcp:8:kworker.0 00000100:00000001:1.0:1787829138.279698:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787829138.279699:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787829138.279700:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829138.279701:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787829138.279703:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829138.279706:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787829138.279707:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601336320 : -126234108215296 : ffff8d30d4d24000) 00000100:00000001:1.0:1787829138.279709:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1787829138.279715:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787829138.279718:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787829138.279719:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829138.279720:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829138.279721:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787829138.279722:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829138.279725:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298611236 00000100:00000001:1.0:1787829138.279726:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298612480 00000100:00000001:1.0:1787829138.279728:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298611236 00000100:00000001:1.0:1787829138.279729:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298612480 00000100:00000001:1.0:1787829138.279730:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829138.279732:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829138.279733:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787829138.279735:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829138/real 1787829138] req@ffff8d30d4d24000 x1874670646800128/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829163 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787829138.279739:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787829138.279740:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787829138.279741:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787829138.279742:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787829138.279743:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829138.279744:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829138.279745:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787829138.279747:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d24000 x1874670646800128/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829163 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787829138.279755:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829138.279757:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787829138.279758:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829138.279759:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829138.279760:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829138.279761:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787829138.279762:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787829138.279763:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829138.279765:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787829138.279766:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787829138.279768:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d24000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646800128:192.168.203.143@tcp:8:kworker.0 00000100:00000001:1.0:1787829138.279770:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829138.279771:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829138.279772:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787829138.279773:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787829138.279774:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787829138.279775:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787829138.279775:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787829138.279777:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787829138.279777:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787829138.279779:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787829138.279780:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829138.279781:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829138.279782:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787829138.279783:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787829138.279784:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787829138.279784:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787829138.279785:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787829138.279786:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787829138.279787:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787829138.279788:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787829138.279789:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829138.279790:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829138.279792:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787829138.279793:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829138.279794:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787829138.279795:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829139.174666:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829139.174674:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829139.174747:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829139.174751:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829139.174754:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829140.198682:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829140.198687:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829140.198717:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829140.198721:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829140.198723:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829141.222690:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829141.222698:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829141.222766:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829141.222770:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829141.222773:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829142.246695:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829142.246700:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829142.246749:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829142.246752:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829142.246755:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829143.270688:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829143.270696:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829143.270763:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829143.270767:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829143.270771:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787829143.398716:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829143.398728:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1787829143.398731:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787829143.398736:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787829143.398741:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787829143.398749:0:468:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 3943 00000100:00080000:0.0:1787829143.398761:0:468:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787829143.398766:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829143.398768:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787829143.398774:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787829143.398777:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787829143.398781:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:0.0:1787829143.398784:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:0.0:1787829143.398786:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829143.398788:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:0.0:1787829143.398809:0:468:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787829143.398812:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829143.398815:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787829143.398819:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787829143.398822:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829143.398825:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1787829143.398828:0:468:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829143.398839:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787829143.398841:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829143.398844:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787829143.398847:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787829143.398851:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829143.398863:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787829143.398874:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d24000 x1874670646800256/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787829143.398886:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787829143.398890:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829143.398922:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787829143.398926:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829143.398930:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1787829143.398932:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787829143.398934:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787829143.398937:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787829143.398939:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3943 00000100:00080000:0.0:1787829143.398944:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787829143.398947:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829143.398949:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787829143.398952:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787829143.398954:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787829143.398956:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000020:00000001:0.0:1787829143.398959:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:0.0:1787829143.398961:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829143.398963:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:1.0:1787829143.398965:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00080000:0.0:1787829143.398967:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:1.0:1787829143.398970:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787829143.398972:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829143.398974:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 02000000:00000001:0.0:1787829143.398978:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1787829143.398978:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787829143.398981:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829143.398986:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829143.398986:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787829143.398990:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787829143.398991:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829143.398994:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829143.398996:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00100000:1.0:1787829143.399000:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d24000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646800256:192.168.203.143@tcp:250:kworker.0 00000008:00100000:0.0:1787829143.399001:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00000001:1.0:1787829143.399007:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000008:00000001:0.0:1787829143.399009:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829143.399013:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829143.399017:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1787829143.399019:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787829143.399020:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829143.399024:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1787829143.399026:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829143.399027:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1787829143.399031:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1787829143.399032:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829143.399036:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829143.399041:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787829143.399051:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d24f00 x1874670646800384/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787829143.399051:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787829143.399054:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601336320 : -126234108215296 : ffff8d30d4d24000) 00000100:00000001:1.0:1787829143.399059:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:0.0:1787829143.399066:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787829143.399068:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829143.399072:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787829143.399075:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829143.399079:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1787829143.399082:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787829143.399084:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787829143.399087:0:468:0:(import.c:485:import_select_connection()) Process entered 00000800:00000100:1.0:1787829143.399090:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00080000:0.0:1787829143.399093:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3943 00000100:00080000:0.0:1787829143.399118:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1787829143.399119:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1787829143.399124:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829143.399126:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:1.0:1787829143.399126:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829143.399130:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787829143.399131:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:1.0:1787829143.399134:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000020:00000001:0.0:1787829143.399135:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787829143.399149:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:1.0:1787829143.399152:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787829143.399153:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:0.0:1787829143.399158:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787829143.399159:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4298617600 00000100:00000001:0.0:1787829143.399160:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:0.0:1787829143.399167:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:1.0:1787829143.399169:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1787829143.399170:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829143.399174:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1787829143.399177:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4298617600 02000000:00000001:0.0:1787829143.399178:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829143.399182:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 02000000:00000001:0.0:1787829143.399183:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829143.399186:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787829143.399187:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:1.0:1787829143.399188:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000008:00100000:0.0:1787829143.399191:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00000001:1.0:1787829143.399192:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000008:00000001:0.0:1787829143.399196:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829143.399199:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829143/real 1787829143] req@ffff8d30d4d24000 x1874670646800256/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829168 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787829143.399200:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829143.399216:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787829143.399219:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829143.399224:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000400:00000001:1.0:1787829143.399225:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 02000000:00000001:0.0:1787829143.399227:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000400:00000001:1.0:1787829143.399228:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 02000000:00000001:0.0:1787829143.399230:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829143.399232:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787829143.399236:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787829143.399238:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829143.399238:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829143.399243:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829143.399245:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787829143.399249:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d25a40 x1874670646800512/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787829143.399251:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d24000 x1874670646800256/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829168 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787829143.399281:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829143.399283:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787829143.399285:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829143.399286:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829143.399288:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829143.399293:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787829143.399293:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787829143.399314:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787829143.399316:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1787829143.399317:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829143.399323:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787829143.399325:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829143.399327:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00080000:0.0:1787829143.399331:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1787829143.399332:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d24000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646800256:192.168.203.143@tcp:250:kworker.0 00000100:00080000:0.0:1787829143.399343:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1787829143.399343:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829143.399346:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787829143.399348:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:1.0:1787829143.399350:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 00000100:00080000:0.0:1787829143.399353:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 02000000:00000001:1.0:1787829143.399354:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 00000100:00000001:0.0:1787829143.399360:0:468:0:(import.c:485:import_select_connection()) Process entered 02000000:00000001:1.0:1787829143.399360:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787829143.399365:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787829143.399368:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 00000100:00080000:0.0:1787829143.399370:0:468:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3943 02000000:00000001:1.0:1787829143.399373:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787829143.399376:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00080000:0.0:1787829143.399377:0:468:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1787829143.399397:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787829143.399402:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787829143.399402:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829143.399408:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:1.0:1787829143.399409:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1787829143.399417:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:1.0:1787829143.399419:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000020:00000001:0.0:1787829143.399421:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:1.0:1787829143.399423:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787829143.399427:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:1.0:1787829143.399428:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787829143.399431:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1787829143.399436:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000020:00000001:0.0:1787829143.399437:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:1.0:1787829143.399438:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787829143.399442:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829143.399463:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 02000000:00000001:1.0:1787829143.399464:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787829143.399467:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787829143.399468:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 02000000:00000001:1.0:1787829143.399470:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829143.399474:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d24f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646800384:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787829143.399477:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00080000:0.0:1787829143.399481:0:468:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 02000000:00000001:1.0:1787829143.399489:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:0.0:1787829143.399491:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829143.399492:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829143.399498:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1787829143.399498:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787829143.399503:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829143.399504:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829143.399507:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 02000000:00000001:0.0:1787829143.399508:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829143.399510:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601340160 : -126234108211456 : ffff8d30d4d24f00) 00000100:00000001:0.0:1787829143.399530:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:1.0:1787829143.399535:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000008:00100000:0.0:1787829143.399536:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1787829143.399543:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829143.399545:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829143.399553:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000800:00000100:1.0:1787829143.399554:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 02000000:00000001:0.0:1787829143.399557:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829143.399562:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1787829143.399563:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:1.0:1787829143.399567:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 02000000:00000001:0.0:1787829143.399568:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:1.0:1787829143.399572:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829143.399574:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829143.399576:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787829143.399581:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829143.399584:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829143.399586:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298616342 00000100:00000001:1.0:1787829143.399593:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298617600 00000100:00000001:1.0:1787829143.399598:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298616342 00000100:00100000:0.0:1787829143.399599:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d270c0 x1874670646800640/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787829143.399601:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298617600 00000100:00000001:1.0:1787829143.399608:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829143.399615:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829143.399618:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787829143.399624:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829143/real 1787829143] req@ffff8d30d4d24f00 x1874670646800384/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829168 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787829143.399633:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787829143.399634:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787829143.399637:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787829143.399639:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787829143.399641:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829143.399661:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829143.399662:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787829143.399669:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d24f00 x1874670646800384/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829168 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787829143.399674:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787829143.399680:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829143.399685:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829143.399720:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829143.399723:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787829143.399725:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829143.399726:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829143.399727:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829143.399730:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787829143.399732:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787829143.399735:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829143.399756:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787829143.399759:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787829143.399763:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d24f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646800384:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787829143.399768:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787829143.399770:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787829143.399771:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829143.399773:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787829143.399775:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829143.399777:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829143.399780:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d25a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646800512:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787829143.399784:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787829143.399786:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787829143.399787:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829143.399789:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787829143.399792:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829143.399798:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787829143.399799:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601343040 : -126234108208576 : ffff8d30d4d25a40) 00000100:00000001:1.0:1787829143.399803:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1787829143.399813:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787829143.399816:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787829143.399818:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829143.399819:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829143.399820:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787829143.399822:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829143.399825:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298616343 00000100:00000001:1.0:1787829143.399826:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298617600 00000100:00000001:1.0:1787829143.399828:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298616343 00000100:00000001:1.0:1787829143.399830:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298617600 00000100:00000001:1.0:1787829143.399831:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829143.399832:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829143.399833:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787829143.399836:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829143/real 1787829143] req@ffff8d30d4d25a40 x1874670646800512/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829168 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787829143.399841:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787829143.399842:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787829143.399843:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787829143.399844:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787829143.399845:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829143.399846:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829143.399847:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787829143.399850:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d25a40 x1874670646800512/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829168 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787829143.399870:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829143.399872:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787829143.399873:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829143.399874:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829143.399875:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829143.399876:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787829143.399877:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787829143.399878:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829143.399880:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787829143.399881:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787829143.399883:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d25a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646800512:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787829143.399885:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829143.399887:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829143.399887:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787829143.399888:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787829143.399890:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787829143.399891:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787829143.399891:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787829143.399893:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787829143.399894:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787829143.399896:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787829143.399897:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829143.399898:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829143.399899:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787829143.399900:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787829143.399901:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787829143.399901:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787829143.399902:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787829143.399903:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787829143.399904:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787829143.399905:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787829143.399906:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829143.399907:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829143.399909:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787829143.399910:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829143.399911:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787829143.399912:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1787829143.399913:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787829143.399914:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787829143.399915:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829143.399917:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787829143.399918:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829143.399919:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829143.399921:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d270c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646800640:192.168.203.143@tcp:8:kworker.0 00000100:00000001:1.0:1787829143.399932:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787829143.399934:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787829143.399935:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829143.399936:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787829143.399937:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829143.399940:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787829143.399941:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601348800 : -126234108202816 : ffff8d30d4d270c0) 00000100:00000001:1.0:1787829143.399944:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1787829143.399950:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787829143.399952:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787829143.399954:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829143.399954:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829143.399956:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787829143.399957:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829143.399959:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298616356 00000100:00000001:1.0:1787829143.399961:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298617600 00000100:00000001:1.0:1787829143.399962:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298616356 00000100:00000001:1.0:1787829143.399963:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298617600 00000100:00000001:1.0:1787829143.399965:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829143.399966:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829143.399967:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787829143.399969:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829143/real 1787829143] req@ffff8d30d4d270c0 x1874670646800640/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829168 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787829143.399978:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787829143.399979:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787829143.399980:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787829143.399981:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787829143.399982:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829143.399983:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829143.399984:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787829143.399986:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d270c0 x1874670646800640/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829168 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787829143.399990:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829143.399991:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787829143.399993:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829143.399994:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829143.399995:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829143.399996:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787829143.399997:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787829143.399999:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829143.400000:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787829143.400002:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787829143.400003:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d270c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646800640:192.168.203.143@tcp:8:kworker.0 00000100:00000001:1.0:1787829143.400005:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829143.400006:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829143.400007:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787829143.400008:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787829143.400009:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787829143.400010:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787829143.400010:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787829143.400012:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787829143.400012:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787829143.400014:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787829143.400015:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829143.400016:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829143.400018:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787829143.400019:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829143.400020:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787829143.400021:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829144.294669:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829144.294677:0:815:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1787829144.294679:0:815:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1787829144.294684:0:815:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829144.294688:0:815:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1787829144.294690:0:815:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829144.294693:0:815:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829144.294695:0:815:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829144.294752:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829144.294756:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829144.294759:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829145.318682:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829145.318688:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829145.318744:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829145.318749:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829145.318752:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829146.342661:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829146.342667:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829146.342715:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829146.342719:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829146.342724:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829147.366698:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829147.366703:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829147.366763:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829147.366767:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829147.366771:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829148.390669:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829148.390674:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829148.390715:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829148.390719:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829148.390721:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787829148.518674:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829148.518684:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1787829148.518687:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787829148.518690:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787829148.518695:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787829148.518703:0:468:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 3949 00000100:00080000:0.0:1787829148.518714:0:468:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787829148.518717:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829148.518719:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787829148.518722:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787829148.518724:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787829148.518727:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:0.0:1787829148.518729:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:0.0:1787829148.518733:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829148.518734:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:0.0:1787829148.518746:0:468:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787829148.518751:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829148.518753:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787829148.518756:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787829148.518758:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829148.518760:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1787829148.518763:0:468:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829148.518770:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787829148.518771:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829148.518774:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787829148.518776:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787829148.518779:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829148.518787:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787829148.518796:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d27c00 x1874670646800768/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787829148.518804:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787829148.518808:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829148.518822:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787829148.518825:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829148.518828:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1787829148.518829:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787829148.518831:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787829148.518834:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787829148.518837:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3949 00000100:00080000:0.0:1787829148.518841:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787829148.518842:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829148.518845:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:3.0:1787829148.518845:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:0.0:1787829148.518849:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:3.0:1787829148.518850:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000020:00000001:0.0:1787829148.518851:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:3.0:1787829148.518854:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000020:00000001:0.0:1787829148.518855:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000020:00000001:0.0:1787829148.518858:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:3.0:1787829148.518858:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787829148.518861:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829148.518862:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829148.518866:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 02000000:00000001:3.0:1787829148.518868:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000100:00080000:0.0:1787829148.518870:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 02000000:00000001:3.0:1787829148.518872:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787829148.518873:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829148.518877:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829148.518878:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787829148.518896:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00100000:3.0:1787829148.518896:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d27c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646800768:192.168.203.143@tcp:250:kworker.0 02000000:00000001:0.0:1787829148.518903:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829148.518904:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:0.0:1787829148.518908:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 02000000:00000001:3.0:1787829148.518909:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1787829148.518913:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000008:00100000:0.0:1787829148.518914:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 02000000:00000001:3.0:1787829148.518915:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000008:00000001:0.0:1787829148.518920:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829148.518921:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829148.518923:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829148.518929:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787829148.518932:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829148.518934:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787829148.518936:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787829148.518939:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829148.518947:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787829148.518965:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d252c0 x1874670646800896/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787829148.518974:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787829148.518976:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829148.518980:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787829148.518983:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829148.518987:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1787829148.518988:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787829148.518990:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787829148.518994:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787829148.518997:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3949 00000100:00080000:0.0:1787829148.519002:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787829148.519005:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829148.519006:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787829148.519009:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787829148.519011:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787829148.519024:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000020:00000001:0.0:1787829148.519027:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:0.0:1787829148.519033:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829148.519035:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:0.0:1787829148.519040:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787829148.519043:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829148.519045:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787829148.519050:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787829148.519052:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829148.519055:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1787829148.519058:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1787829148.519061:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829148.519063:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829148.519080:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787829148.519082:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829148.519084:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787829148.519085:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787829148.519087:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829148.519092:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787829148.519098:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d25680 x1874670646801024/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787829148.519107:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787829148.519109:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829148.519112:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787829148.519115:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829148.519118:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1787829148.519120:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787829148.519132:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787829148.519134:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787829148.519138:0:468:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3949 00000100:00080000:0.0:1787829148.519142:0:468:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787829148.519145:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829148.519146:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787829148.519149:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:3.0:1787829148.519151:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000020:00000001:0.0:1787829148.519152:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:3.0:1787829148.519154:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601351680 : -126234108199936 : ffff8d30d4d27c00) 00000020:00000001:0.0:1787829148.519156:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:3.0:1787829148.519159:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000020:00000001:0.0:1787829148.519161:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:0.0:1787829148.519166:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829148.519168:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:0.0:1787829148.519172:0:468:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787829148.519176:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829148.519179:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787829148.519183:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787829148.519185:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829148.519188:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1787829148.519191:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1787829148.519194:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829148.519196:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829148.519201:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787829148.519203:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829148.519206:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787829148.519207:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787829148.519210:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829148.519216:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787829148.519224:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d26940 x1874670646801152/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1787829148.519234:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:0.0:1787829148.519237:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787829148.519241:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829148.519245:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829148.519246:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787829148.519252:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829148.519254:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829148.519256:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1787829148.519262:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829148.519265:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4298622720 00000100:00000001:3.0:1787829148.519273:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 00000100:00000001:3.0:1787829148.519279:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4298622720 00000100:00000001:3.0:1787829148.519282:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 00000100:00000001:3.0:1787829148.519284:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829148.519286:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829148.519289:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1787829148.519293:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829148/real 1787829148] req@ffff8d30d4d27c00 x1874670646800768/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829173 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1787829148.519301:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787829148.519303:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829148.519305:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829148.519307:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829148.519308:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829148.519310:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829148.519312:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829148.519315:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d27c00 x1874670646800768/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829173 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829148.519320:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829148.519322:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829148.519325:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829148.519326:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829148.519326:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829148.519328:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829148.519329:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829148.519331:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829148.519334:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829148.519336:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829148.519338:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d27c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646800768:192.168.203.143@tcp:250:kworker.0 00000100:00000001:3.0:1787829148.519342:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829148.519343:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829148.519344:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829148.519346:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829148.519348:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829148.519349:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829148.519351:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829148.519353:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829148.519354:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829148.519357:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829148.519358:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829148.519359:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829148.519365:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787829148.519366:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829148.519368:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1787829148.519369:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787829148.519370:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787829148.519371:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787829148.519372:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829148.519373:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787829148.519374:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829148.519376:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829148.519378:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d252c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646800896:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829148.519380:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1787829148.519381:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1787829148.519383:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829148.519384:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1787829148.519386:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829148.519390:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787829148.519392:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601341120 : -126234108210496 : ffff8d30d4d252c0) 00000100:00000001:3.0:1787829148.519396:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1787829148.519409:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:3.0:1787829148.519423:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787829148.519426:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829148.519428:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829148.519430:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1787829148.519433:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829148.519438:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298621462 00000100:00000001:3.0:1787829148.519441:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298622720 00000100:00000001:3.0:1787829148.519444:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298621462 00000100:00000001:3.0:1787829148.519447:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298622720 00000100:00000001:3.0:1787829148.519449:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829148.519451:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829148.519453:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1787829148.519458:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829148/real 1787829148] req@ffff8d30d4d252c0 x1874670646800896/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829173 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1787829148.519474:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787829148.519476:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829148.519479:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829148.519481:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829148.519483:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829148.519485:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829148.519486:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829148.519491:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d252c0 x1874670646800896/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829173 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829148.519499:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829148.519502:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829148.519504:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829148.519506:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829148.519507:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829148.519509:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829148.519510:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829148.519512:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829148.519515:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829148.519517:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829148.519519:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d252c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646800896:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829148.519522:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787829148.519523:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787829148.519525:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829148.519526:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787829148.519527:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829148.519529:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829148.519531:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d25680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646801024:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829148.519533:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1787829148.519534:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1787829148.519535:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829148.519536:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1787829148.519539:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829148.519543:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787829148.519544:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601342080 : -126234108209536 : ffff8d30d4d25680) 00000100:00000001:3.0:1787829148.519547:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1787829148.519557:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:3.0:1787829148.519560:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787829148.519562:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829148.519563:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829148.519564:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1787829148.519566:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829148.519568:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298621462 00000100:00000001:3.0:1787829148.519570:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298622720 00000100:00000001:3.0:1787829148.519572:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298621462 00000100:00000001:3.0:1787829148.519573:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298622720 00000100:00000001:3.0:1787829148.519575:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829148.519576:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829148.519577:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1787829148.519580:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829148/real 1787829148] req@ffff8d30d4d25680 x1874670646801024/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829173 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1787829148.519590:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787829148.519591:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829148.519592:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829148.519593:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829148.519594:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829148.519595:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829148.519596:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829148.519598:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d25680 x1874670646801024/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829173 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829148.519625:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829148.519627:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829148.519629:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829148.519630:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829148.519631:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829148.519632:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829148.519634:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829148.519636:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829148.519639:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829148.519642:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829148.519645:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d25680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646801024:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829148.519649:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787829148.519651:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787829148.519652:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829148.519654:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787829148.519656:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829148.519657:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829148.519659:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d26940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646801152:192.168.203.143@tcp:8:kworker.0 00000100:00000001:3.0:1787829148.519662:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1787829148.519663:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1787829148.519665:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829148.519666:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1787829148.519668:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829148.519671:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787829148.519672:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601346880 : -126234108204736 : ffff8d30d4d26940) 00000100:00000001:3.0:1787829148.519675:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1787829148.519682:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:3.0:1787829148.519685:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787829148.519687:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829148.519688:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829148.519689:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1787829148.519691:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829148.519693:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298621476 00000100:00000001:3.0:1787829148.519694:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298622720 00000100:00000001:3.0:1787829148.519696:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298621476 00000100:00000001:3.0:1787829148.519697:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298622720 00000100:00000001:3.0:1787829148.519699:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829148.519700:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829148.519701:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1787829148.519703:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829148/real 1787829148] req@ffff8d30d4d26940 x1874670646801152/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829173 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1787829148.519710:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787829148.519711:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829148.519713:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829148.519715:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829148.519716:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829148.519718:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829148.519719:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829148.519724:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d26940 x1874670646801152/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829173 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829148.519731:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829148.519733:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829148.519736:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829148.519738:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829148.519740:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829148.519741:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829148.519743:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829148.519746:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829148.519749:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829148.519752:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829148.519755:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d26940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646801152:192.168.203.143@tcp:8:kworker.0 00000100:00000001:3.0:1787829148.519759:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829148.519761:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829148.519762:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829148.519764:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829148.519766:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829148.519768:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829148.519769:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829148.519772:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829148.519773:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829148.519775:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829148.519777:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829148.519779:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829148.519780:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829148.519782:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829148.519783:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829148.519784:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829148.519785:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829148.519788:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829148.519789:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829148.519791:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829148.519792:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829148.519794:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829148.519794:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829148.519795:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829148.519796:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829148.519797:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829148.519798:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829148.519799:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829148.519801:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829148.519803:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829148.519803:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829148.519805:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829148.519807:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787829148.519808:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829148.519810:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1787829148.519811:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829149.414685:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829149.414691:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829149.414779:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829149.414783:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829149.414786:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829150.438694:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829150.438699:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829150.438758:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829150.438762:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829150.438764:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829151.462698:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829151.462703:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829151.462782:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829151.462786:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829151.462788:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829152.486769:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829152.486783:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829152.486880:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829152.486885:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829152.486889:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829153.510752:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829153.510763:0:815:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1787829153.510766:0:815:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1787829153.510770:0:815:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829153.510779:0:815:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1787829153.510783:0:815:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829153.510786:0:815:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829153.510806:0:815:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829153.510870:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829153.510891:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829153.510896:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787829153.638709:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829153.638723:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1787829153.638726:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787829153.638731:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787829153.638737:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787829153.638754:0:468:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 3954 00000100:00080000:0.0:1787829153.638767:0:468:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787829153.638772:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829153.638774:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787829153.638779:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787829153.638782:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787829153.638786:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:0.0:1787829153.638788:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:0.0:1787829153.638791:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829153.638792:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:0.0:1787829153.638798:0:468:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787829153.638801:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829153.638804:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787829153.638808:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787829153.638812:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829153.638814:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1787829153.638817:0:468:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829153.638832:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787829153.638834:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829153.638838:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787829153.638841:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787829153.638847:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829153.638864:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787829153.638876:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d26940 x1874670646801280/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787829153.638888:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787829153.638893:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829153.638910:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787829153.638913:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829153.638917:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1787829153.638919:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787829153.638921:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787829153.638924:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787829153.638926:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3954 00000100:00080000:0.0:1787829153.638930:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787829153.638933:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829153.638935:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787829153.638938:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787829153.638940:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787829153.638942:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000020:00000001:0.0:1787829153.638945:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:0.0:1787829153.638947:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829153.638950:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:0.0:1787829153.638954:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787829153.638957:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829153.638959:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787829153.638966:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787829153.638968:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829153.638970:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:3.0:1787829153.638972:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000008:00100000:0.0:1787829153.638976:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00000001:3.0:1787829153.638979:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000008:00000001:0.0:1787829153.638982:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829153.638983:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1787829153.638988:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829153.638988:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787829153.638992:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829153.638995:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787829153.638997:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787829153.638999:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829153.639001:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787829153.639003:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787829153.639004:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829153.639007:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787829153.639012:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829153.639012:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d26940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646801280:192.168.203.143@tcp:250:kworker.0 00000100:00000001:3.0:1787829153.639019:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:0.0:1787829153.639021:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829153.639023:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1787829153.639027:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829153.639041:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00100000:0.0:1787829153.639044:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d25680 x1874670646801408/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:3.0:1787829153.639047:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829153.639058:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787829153.639061:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829153.639065:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829153.639068:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00080000:0.0:1787829153.639069:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1787829153.639071:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601346880 : -126234108204736 : ffff8d30d4d26940) 00000100:00080000:0.0:1787829153.639078:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1787829153.639087:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:3.0:1787829153.639088:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00080000:0.0:1787829153.639090:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787829153.639093:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787829153.639096:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3954 00000100:00080000:0.0:1787829153.639100:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787829153.639102:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829153.639104:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787829153.639107:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787829153.639109:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787829153.639111:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000020:00000001:0.0:1787829153.639114:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:0.0:1787829153.639116:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829153.639118:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:0.0:1787829153.639121:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787829153.639123:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829153.639126:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787829153.639129:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787829153.639134:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:3.0:1787829153.639134:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:0.0:1787829153.639139:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1787829153.639143:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1787829153.639146:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829153.639148:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829153.639152:0:2390:0:(events.c:34:request_out_callback()) Process entered 02000000:00000001:0.0:1787829153.639154:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787829153.639157:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829153.639158:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787829153.639160:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:3.0:1787829153.639162:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829153.639164:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:3.0:1787829153.639165:0:2390:0:(events.c:68:request_out_callback()) Process leaving 02000000:00000001:0.0:1787829153.639168:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829153.639172:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829153.639176:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829153.639177:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4298627840 00000100:00100000:0.0:1787829153.639185:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d252c0 x1874670646801536/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787829153.639189:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1787829153.639198:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787829153.639199:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4298627840 00000100:00000001:0.0:1787829153.639202:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829153.639205:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1787829153.639207:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829153.639209:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00080000:0.0:1787829153.639211:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1787829153.639212:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829153.639217:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00080000:0.0:1787829153.639218:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1787829153.639221:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787829153.639225:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1787829153.639225:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829153/real 1787829153] req@ffff8d30d4d26940 x1874670646801280/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829178 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787829153.639232:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787829153.639238:0:468:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3954 00000400:00000001:3.0:1787829153.639240:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787829153.639243:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00080000:0.0:1787829153.639244:0:468:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787829153.639246:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787829153.639249:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829153.639250:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787829153.639255:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:3.0:1787829153.639255:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829153.639259:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000020:00000001:0.0:1787829153.639261:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:3.0:1787829153.639262:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1787829153.639264:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787829153.639269:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00100000:3.0:1787829153.639269:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d26940 x1874670646801280/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829178 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:0.0:1787829153.639275:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:0.0:1787829153.639282:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829153.639282:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829153.639286:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:3.0:1787829153.639287:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829153.639291:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:0.0:1787829153.639294:0:468:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:3.0:1787829153.639294:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829153.639296:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829153.639301:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829153.639302:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829153.639305:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 02000000:00000001:0.0:1787829153.639306:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00080000:3.0:1787829153.639308:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 02000000:00000001:0.0:1787829153.639311:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00080000:3.0:1787829153.639312:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 02000000:00000001:0.0:1787829153.639317:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829153.639317:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000001:0.0:1787829153.639323:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00100000:3.0:1787829153.639324:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d26940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646801280:192.168.203.143@tcp:250:kworker.0 00000008:00100000:0.0:1787829153.639327:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00000001:3.0:1787829153.639330:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000008:00000001:0.0:1787829153.639333:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829153.639333:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829153.639336:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 00000100:00000001:0.0:1787829153.639337:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829153.639340:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829153.639343:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 02000000:00000001:0.0:1787829153.639344:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000020:00000001:3.0:1787829153.639345:0:2390:0:(genops.c:1217:class_import_put()) Process entered 02000000:00000001:0.0:1787829153.639348:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829153.639349:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 00000100:00000001:0.0:1787829153.639353:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787829153.639355:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787829153.639356:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787829153.639359:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 02000000:00000001:0.0:1787829153.639360:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829153.639363:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829153.639365:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829153.639367:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829153.639367:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829153.639378:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00100000:0.0:1787829153.639379:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d27c00 x1874670646801664/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787829153.639381:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829153.639386:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1787829153.639389:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787829153.639391:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787829153.639393:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787829153.639396:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829153.639397:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 02000000:00000001:3.0:1787829153.639398:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787829153.639401:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787829153.639402:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829153.639404:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829153.639408:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829153.639409:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d25680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646801408:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829153.639414:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1787829153.639416:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1787829153.639418:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829153.639419:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1787829153.639422:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829153.639427:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787829153.639428:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601342080 : -126234108209536 : ffff8d30d4d25680) 00000100:00000001:3.0:1787829153.639432:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1787829153.639441:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:3.0:1787829153.639444:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787829153.639446:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829153.639447:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829153.639449:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1787829153.639451:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829153.639455:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298626582 00000100:00000001:3.0:1787829153.639457:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298627840 00000100:00000001:3.0:1787829153.639459:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298626582 00000100:00000001:3.0:1787829153.639460:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298627840 00000100:00000001:3.0:1787829153.639462:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829153.639463:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829153.639464:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1787829153.639467:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829153/real 1787829153] req@ffff8d30d4d25680 x1874670646801408/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829178 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1787829153.639472:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787829153.639473:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829153.639474:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829153.639475:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829153.639476:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829153.639477:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829153.639478:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829153.639481:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d25680 x1874670646801408/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829178 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829153.639485:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829153.639486:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829153.639488:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829153.639489:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829153.639490:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829153.639491:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829153.639492:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829153.639493:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829153.639495:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829153.639496:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829153.639498:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d25680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646801408:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829153.639503:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787829153.639504:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787829153.639505:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829153.639507:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787829153.639508:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829153.639509:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829153.639511:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d252c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646801536:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829153.639512:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1787829153.639513:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1787829153.639515:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829153.639516:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1787829153.639518:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829153.639520:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787829153.639521:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601341120 : -126234108210496 : ffff8d30d4d252c0) 00000100:00000001:3.0:1787829153.639524:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1787829153.639530:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:3.0:1787829153.639532:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787829153.639533:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829153.639534:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829153.639535:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1787829153.639537:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829153.639539:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298626582 00000100:00000001:3.0:1787829153.639541:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298627840 00000100:00000001:3.0:1787829153.639543:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298626582 00000100:00000001:3.0:1787829153.639544:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298627840 00000100:00000001:3.0:1787829153.639546:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829153.639547:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829153.639548:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1787829153.639550:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829153/real 1787829153] req@ffff8d30d4d252c0 x1874670646801536/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829178 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1787829153.639555:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787829153.639556:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829153.639557:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829153.639558:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829153.639559:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829153.639560:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829153.639561:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829153.639564:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d252c0 x1874670646801536/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829178 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829153.639568:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829153.639569:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829153.639570:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829153.639571:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829153.639572:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829153.639573:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829153.639574:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829153.639575:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829153.639577:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829153.639578:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829153.639580:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d252c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646801536:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829153.639582:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829153.639583:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829153.639584:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829153.639585:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829153.639586:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829153.639587:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829153.639587:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829153.639589:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829153.639590:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829153.639592:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829153.639593:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829153.639594:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829153.639595:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829153.639595:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829153.639597:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829153.639598:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829153.639622:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829153.639624:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829153.639625:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829153.639626:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829153.639627:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829153.639629:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829153.639631:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787829153.639632:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829153.639634:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1787829153.639635:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787829153.639636:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787829153.639637:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787829153.639638:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829153.639640:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787829153.639641:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829153.639643:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829153.639645:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d27c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646801664:192.168.203.143@tcp:8:kworker.0 00000100:00000001:3.0:1787829153.639647:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1787829153.639649:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1787829153.639650:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829153.639651:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1787829153.639653:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829153.639664:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787829153.639665:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601351680 : -126234108199936 : ffff8d30d4d27c00) 00000100:00000001:3.0:1787829153.639668:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1787829153.639676:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:3.0:1787829153.639679:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787829153.639680:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829153.639681:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829153.639683:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1787829153.639684:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829153.639686:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298626596 00000100:00000001:3.0:1787829153.639688:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298627840 00000100:00000001:3.0:1787829153.639690:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298626596 00000100:00000001:3.0:1787829153.639691:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298627840 00000100:00000001:3.0:1787829153.639693:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829153.639695:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829153.639696:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1787829153.639699:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829153/real 1787829153] req@ffff8d30d4d27c00 x1874670646801664/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829178 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1787829153.639704:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787829153.639705:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829153.639707:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829153.639708:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829153.639709:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829153.639710:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829153.639711:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829153.639713:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d27c00 x1874670646801664/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829178 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829153.639718:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829153.639719:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829153.639720:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829153.639721:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829153.639722:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829153.639723:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829153.639724:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829153.639726:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829153.639727:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829153.639728:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829153.639730:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d27c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646801664:192.168.203.143@tcp:8:kworker.0 00000100:00000001:3.0:1787829153.639732:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829153.639733:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829153.639734:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829153.639735:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829153.639737:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829153.639737:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829153.639739:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829153.639740:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829153.639741:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829153.639744:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829153.639745:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829153.639746:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829153.639747:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787829153.639748:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829153.639750:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1787829153.639751:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829154.534701:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829154.534711:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829154.534804:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829154.534808:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829154.534898:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829155.558699:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829155.558707:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829155.558791:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829155.558796:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829155.558799:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829156.582710:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829156.582716:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829156.582773:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829156.582777:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829156.582780:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829157.606673:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829157.606679:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829157.606759:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829157.606764:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829157.606768:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829158.630714:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829158.630719:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829158.630781:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829158.630786:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829158.630791:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787829158.758850:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829158.758861:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1787829158.758864:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787829158.758867:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787829158.758872:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787829158.758880:0:40798:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 3959 00000100:00080000:2.0:1787829158.758891:0:40798:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787829158.758895:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829158.758897:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787829158.758917:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787829158.758920:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787829158.758923:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:2.0:1787829158.758924:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:2.0:1787829158.758927:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829158.758928:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:2.0:1787829158.758932:0:40798:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787829158.758934:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829158.758937:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787829158.758940:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787829158.758943:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829158.758945:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:2.0:1787829158.758948:0:40798:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829158.758963:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787829158.758965:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829158.758968:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787829158.758971:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787829158.758976:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829158.758991:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787829158.759015:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2e940 x1874670646801792/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787829158.759048:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787829158.759052:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829158.759069:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787829158.759072:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829158.759075:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1787829158.759077:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787829158.759078:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787829158.759080:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787829158.759082:0:40798:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3959 00000100:00080000:2.0:1787829158.759085:0:40798:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787829158.759086:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829158.759087:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787829158.759089:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787829158.759090:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787829158.759092:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000020:00000001:2.0:1787829158.759095:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:2.0:1787829158.759096:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829158.759097:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:2.0:1787829158.759100:0:40798:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787829158.759102:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829158.759103:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787829158.759110:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787829158.759112:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829158.759114:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:3.0:1787829158.759115:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000008:00100000:2.0:1787829158.759121:0:40798:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00000001:3.0:1787829158.759121:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000008:00000001:2.0:1787829158.759127:0:40798:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829158.759127:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1787829158.759132:0:40798:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829158.759133:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787829158.759137:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829158.759138:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787829158.759141:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829158.759143:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:2.0:1787829158.759144:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787829158.759148:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787829158.759148:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787829158.759150:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829158.759153:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829158.759157:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829158.759160:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646801792:192.168.203.143@tcp:250:kworker.0 00000100:00100000:2.0:1787829158.759165:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2f0c0 x1874670646801920/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787829158.759168:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1787829158.759173:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1787829158.759176:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829158.759181:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 02000000:00000001:3.0:1787829158.759182:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:2.0:1787829158.759188:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829158.759190:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829158.759193:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787829158.759198:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829158.759202:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1787829158.759204:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787829158.759206:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787829158.759209:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787829158.759210:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:3.0:1787829158.759212:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417878848 : -126234291672768 : ffff8d30c9e2e940) 00000100:00080000:2.0:1787829158.759217:0:40798:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3959 00000100:00000001:3.0:1787829158.759219:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00080000:2.0:1787829158.759226:0:40798:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787829158.759229:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829158.759231:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787829158.759234:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787829158.759236:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787829158.759239:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000020:00000001:2.0:1787829158.759241:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:2.0:1787829158.759244:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829158.759245:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:2.0:1787829158.759249:0:40798:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787829158.759253:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829158.759255:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787829158.759259:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787829158.759262:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829158.759264:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1787829158.759266:0:40798:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1787829158.759269:0:40798:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829158.759272:0:40798:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829158.759277:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787829158.759278:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829158.759280:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787829158.759282:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:3.0:1787829158.759283:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829158.759285:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829158.759289:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4298632960 00000100:00000001:2.0:1787829158.759293:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787829158.759301:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2ed00 x1874670646802048/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787829158.759303:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 00000100:00000001:2.0:1787829158.759313:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787829158.759314:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4298632960 00000100:00000001:2.0:1787829158.759318:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829158.759320:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 00000100:00000001:3.0:1787829158.759326:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787829158.759327:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829158.759329:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787829158.759332:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1787829158.759334:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829158.759338:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:2.0:1787829158.759341:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1787829158.759345:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787829158.759347:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787829158.759348:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1787829158.759352:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:3.0:1787829158.759352:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:3.0:1787829158.759373:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00080000:2.0:1787829158.759375:0:40798:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3959 00000100:00000001:3.0:1787829158.759376:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787829158.759379:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787829158.759382:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787829158.759383:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787829158.759385:0:40798:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 02000000:00000001:3.0:1787829158.759386:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787829158.759390:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787829158.759391:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829158.759393:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787829158.759396:0:40798:0:(genops.c:792:class_conn2export()) Process entered 02000000:00000001:3.0:1787829158.759396:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1787829158.759399:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787829158.759402:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000020:00000001:2.0:1787829158.759406:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00100000:3.0:1787829158.759406:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646801920:192.168.203.143@tcp:38:kworker.0 00000100:00000001:2.0:1787829158.759408:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829158.759410:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:2.0:1787829158.759416:0:40798:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:3.0:1787829158.759417:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:2.0:1787829158.759418:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829158.759420:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787829158.759423:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787829158.759424:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829158.759426:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1787829158.759429:0:40798:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 02000000:00000001:3.0:1787829158.759430:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000008:00000001:2.0:1787829158.759431:0:40798:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829158.759434:0:40798:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829158.759435:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829158.759438:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787829158.759439:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1787829158.759441:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829158.759443:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787829158.759445:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787829158.759446:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829158.759447:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829158.759456:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787829158.759460:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2c000 x1874670646802176/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787829158.759460:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787829158.759464:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417880768 : -126234291670848 : ffff8d30c9e2f0c0) 00000100:00000001:2.0:1787829158.759471:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787829158.759474:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829158.759474:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:2.0:1787829158.759479:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829158.759495:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829158.759502:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298631702 00000100:00000001:3.0:1787829158.759505:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298632960 00000100:00000001:3.0:1787829158.759508:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298631702 00000100:00000001:3.0:1787829158.759510:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298632960 00000100:00000001:3.0:1787829158.759512:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829158.759513:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829158.759516:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787829158.759517:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787829158.759518:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829158.759520:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787829158.759522:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829158.759524:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829158.759527:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646802048:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829158.759530:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1787829158.759532:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1787829158.759533:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829158.759534:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1787829158.759536:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829158.759541:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787829158.759543:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417879808 : -126234291671808 : ffff8d30c9e2ed00) 00000100:00000001:3.0:1787829158.759546:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:3.0:1787829158.759557:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829158.759562:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298631702 00000100:00000001:3.0:1787829158.759564:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298632960 00000100:00000001:3.0:1787829158.759567:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298631702 00000100:00000001:3.0:1787829158.759569:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298632960 00000100:00000001:3.0:1787829158.759571:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829158.759574:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829158.759576:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829158.759578:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829158.759582:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787829158.759584:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:3.0:1787829158.759586:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1787829158.759588:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787829158.759589:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787829158.759590:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787829158.759591:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829158.759593:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787829158.759595:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829158.759596:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829158.759622:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646802176:192.168.203.143@tcp:8:kworker.0 00000100:00000001:3.0:1787829158.759625:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1787829158.759627:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1787829158.759628:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829158.759629:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1787829158.759632:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829158.759638:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787829158.759639:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417868288 : -126234291683328 : ffff8d30c9e2c000) 00000100:00000001:3.0:1787829158.759642:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:3.0:1787829158.759651:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829158.759654:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298631716 00000100:00000001:3.0:1787829158.759655:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298632960 00000100:00000001:3.0:1787829158.759657:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298631716 00000100:00000001:3.0:1787829158.759659:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298632960 00000100:00000001:3.0:1787829158.759660:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829158.759662:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829158.759663:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829158.759665:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829158.759666:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787829158.759667:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:3.0:1787829158.759668:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1787829158.759669:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787829158.759670:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829158.759672:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829159.654788:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829159.654799:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829159.654891:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829159.654894:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829159.654897:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829160.678792:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829160.678801:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829160.678912:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829160.678915:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829160.678919:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829161.702754:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829161.702767:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829161.702866:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829161.702872:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829161.702876:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:1.0:1787829161.830895:0:2380:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.143:988 00000400:00000100:1.0:1787829161.830946:0:2380:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.143@tcp at host 192.168.203.143 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1787829161.830965:0:2380:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.143@tcp 00000100:00000001:1.0:1787829161.830999:0:2380:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787829161.831022:0:2380:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829161.831026:0:2380:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829161.831029:0:2380:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:1.0:1787829161.831043:0:2380:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.143@tcp 00000100:00000001:1.0:1787829161.831050:0:2380:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787829161.831054:0:2380:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829161.831056:0:2380:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829161.831059:0:2380:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:1.0:1787829161.831065:0:2380:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.143@tcp 00000100:00000001:3.0:1787829161.831075:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1787829161.831081:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787829161.831085:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00000001:1.0:1787829161.831087:0:2380:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787829161.831093:0:2380:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829161.831095:0:2380:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829161.831099:0:2380:0:(events.c:68:request_out_callback()) Process leaving 00000100:00100000:3.0:1787829161.831101:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829158/real 1787829161] req@ffff8d30c9e2f0c0 x1874670646801920/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829183 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000100:1.0:1787829161.831111:0:2380:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.143@tcp 00000100:00000001:1.0:1787829161.831123:0:2380:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787829161.831128:0:2380:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829161.831130:0:2380:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:3.0:1787829161.831131:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000001:1.0:1787829161.831133:0:2380:0:(events.c:68:request_out_callback()) Process leaving 00000400:00000001:3.0:1787829161.831143:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829161.831154:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829161.831158:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829161.831161:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829161.831166:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829161.831169:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829161.831176:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2f0c0 x1874670646801920/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829183 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829161.831185:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829161.831190:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829161.831195:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829161.831197:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829161.831199:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829161.831203:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829161.831207:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829161.831212:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829161.831219:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829161.831223:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829161.831243:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646801920:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829161.831251:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1787829161.831257:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829158/real 1787829161] req@ffff8d30c9e2ed00 x1874670646802048/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829183 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1787829161.831269:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787829161.831271:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829161.831276:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829161.831278:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829161.831280:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829161.831283:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829161.831286:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829161.831291:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2ed00 x1874670646802048/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829183 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829161.831301:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829161.831303:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829161.831306:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829161.831309:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829161.831310:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829161.831312:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829161.831314:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829161.831318:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829161.831323:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829161.831326:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829161.831331:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646802048:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829161.831336:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1787829161.831342:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829158/real 1787829161] req@ffff8d30c9e2e940 x1874670646801792/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829183 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1787829161.831358:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787829161.831360:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829161.831365:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829161.831368:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829161.831369:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829161.831372:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829161.831373:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829161.831379:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2e940 x1874670646801792/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829183 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829161.831388:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829161.831390:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829161.831393:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829161.831396:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829161.831398:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829161.831400:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829161.831402:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829161.831405:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829161.831409:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829161.831411:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829161.831415:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646801792:192.168.203.143@tcp:250:kworker.0 00000100:00000001:3.0:1787829161.831420:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829161.831423:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829161.831426:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829161.831430:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829161.831437:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829161.831441:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829161.831444:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829161.831449:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829161.831451:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829161.831456:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829161.831458:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829161.831460:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829161.831462:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829161.831463:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829161.831465:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829161.831467:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829161.831468:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829161.831470:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829161.831471:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829161.831476:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829161.831477:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829161.831479:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829161.831480:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829161.831481:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829161.831483:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829161.831485:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829161.831487:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829161.831489:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829161.831490:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829161.831491:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829161.831494:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829161.831497:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829161.831499:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1787829161.831501:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787829161.831503:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1787829161.831509:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829158/real 1787829161] req@ffff8d30c9e2c000 x1874670646802176/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829183 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1787829161.831519:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787829161.831520:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829161.831524:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829161.831526:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829161.831528:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829161.831530:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829161.831532:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829161.831536:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2c000 x1874670646802176/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829183 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829161.831544:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829161.831546:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829161.831549:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829161.831550:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829161.831552:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829161.831554:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829161.831556:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829161.831559:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829161.831564:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829161.831566:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829161.831570:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646802176:192.168.203.143@tcp:8:kworker.0 00000100:00000001:3.0:1787829161.831575:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829161.831578:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829161.831579:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829161.831581:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829161.831584:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829161.831587:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829161.831588:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829161.831591:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829161.831592:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829161.831595:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829161.831597:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829161.831728:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829161.831740:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787829161.831743:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829161.831746:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1787829161.831747:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829162.726726:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829162.726737:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829162.726823:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829162.726830:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829162.726834:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829163.750746:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829163.750755:0:815:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1787829163.750757:0:815:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1787829163.750761:0:815:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829163.750767:0:815:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1787829163.750770:0:815:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829163.750772:0:815:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829163.750776:0:815:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829163.750825:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829163.750831:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829163.750835:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787829163.878717:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829163.878727:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1787829163.878731:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787829163.878735:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787829163.878742:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787829163.878756:0:40798:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 3964 00000100:00080000:0.0:1787829163.878766:0:40798:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787829163.878770:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829163.878772:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787829163.878776:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787829163.878779:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787829163.878782:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:0.0:1787829163.878785:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:0.0:1787829163.878787:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829163.878788:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:0.0:1787829163.878793:0:40798:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787829163.878796:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829163.878798:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787829163.878801:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787829163.878804:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829163.878807:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1787829163.878809:0:40798:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829163.878819:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787829163.878821:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829163.878825:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787829163.878828:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787829163.878832:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829163.878842:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787829163.878852:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d270c0 x1874670646802304/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787829163.878863:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787829163.878868:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829163.878888:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787829163.878891:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829163.878895:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1787829163.878896:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787829163.878898:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787829163.878902:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787829163.878905:0:40798:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3964 00000100:00080000:0.0:1787829163.878909:0:40798:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787829163.878911:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829163.878913:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787829163.878916:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787829163.878918:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787829163.878920:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000020:00000001:0.0:1787829163.878923:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:0.0:1787829163.878926:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829163.878927:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:0.0:1787829163.878932:0:40798:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787829163.878935:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829163.878935:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:0.0:1787829163.878940:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:3.0:1787829163.878942:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787829163.878947:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787829163.878950:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 02000000:00000001:0.0:1787829163.878952:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829163.878953:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829163.878974:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787829163.878975:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829163.878979:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787829163.878980:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 02000000:00000001:3.0:1787829163.878986:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000008:00100000:0.0:1787829163.878987:0:40798:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1787829163.878993:0:40798:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829163.878994:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d270c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646802304:192.168.203.143@tcp:250:kworker.0 00000100:00000001:0.0:1787829163.878996:0:40798:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829163.879004:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787829163.879005:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787829163.879008:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829163.879010:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:0.0:1787829163.879011:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787829163.879015:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787829163.879016:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829163.879017:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829163.879022:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:0.0:1787829163.879026:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829163.879030:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787829163.879035:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d25a40 x1874670646802432/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787829163.879046:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787829163.879049:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829163.879050:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787829163.879053:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829163.879054:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601348800 : -126234108202816 : ffff8d30d4d270c0) 00000100:00080000:0.0:1787829163.879060:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1787829163.879067:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00080000:0.0:1787829163.879073:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1787829163.879075:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787829163.879077:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787829163.879080:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787829163.879083:0:40798:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3964 00000100:00080000:0.0:1787829163.879088:0:40798:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787829163.879091:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829163.879092:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787829163.879095:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787829163.879097:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787829163.879099:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000020:00000001:0.0:1787829163.879102:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:0.0:1787829163.879104:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829163.879106:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:0.0:1787829163.879110:0:40798:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787829163.879116:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:3.0:1787829163.879116:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 02000000:00000001:0.0:1787829163.879122:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787829163.879125:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787829163.879128:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829163.879131:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1787829163.879134:0:40798:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00000001:3.0:1787829163.879136:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000008:00000001:0.0:1787829163.879137:0:40798:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829163.879141:0:40798:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829163.879143:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 02000000:00000001:0.0:1787829163.879147:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:3.0:1787829163.879148:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829163.879151:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829163.879154:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787829163.879155:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787829163.879158:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787829163.879160:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829163.879162:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829163.879167:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829163.879169:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4298638080 00000100:00100000:0.0:1787829163.879177:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d24f00 x1874670646802560/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787829163.879183:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1787829163.879195:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787829163.879197:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829163.879199:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4298638080 00000100:00000001:0.0:1787829163.879201:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829163.879208:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 00000100:00080000:0.0:1787829163.879211:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1787829163.879214:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00080000:0.0:1787829163.879216:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1787829163.879220:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:3.0:1787829163.879220:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787829163.879224:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787829163.879226:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00000001:0.0:1787829163.879228:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787829163.879235:0:40798:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3964 00000100:00100000:3.0:1787829163.879237:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829163/real 1787829163] req@ffff8d30d4d270c0 x1874670646802304/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829188 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829163.879239:0:40798:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787829163.879244:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829163.879246:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787829163.879251:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787829163.879254:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787829163.879258:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000400:00000001:3.0:1787829163.879261:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000020:00000001:0.0:1787829163.879262:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000400:00000001:3.0:1787829163.879265:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787829163.879266:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829163.879270:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:3.0:1787829163.879273:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00080000:0.0:1787829163.879275:0:40798:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:3.0:1787829163.879279:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787829163.879280:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829163.879284:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:3.0:1787829163.879284:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:0.0:1787829163.879288:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829163.879289:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 02000000:00000001:0.0:1787829163.879292:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829163.879294:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829163.879296:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1787829163.879300:0:40798:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1787829163.879304:0:40798:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829163.879305:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d270c0 x1874670646802304/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829188 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787829163.879307:0:40798:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829163.879312:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787829163.879315:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829163.879320:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787829163.879324:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:3.0:1787829163.879325:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:0.0:1787829163.879326:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829163.879331:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787829163.879333:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829163.879337:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829163.879342:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00100000:0.0:1787829163.879344:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d24000 x1874670646802688/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787829163.879345:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829163.879351:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829163.879355:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787829163.879360:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00080000:3.0:1787829163.879362:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00000001:0.0:1787829163.879365:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829163.879369:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787829163.879371:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829163.879376:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829163.879382:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d270c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646802304:192.168.203.143@tcp:250:kworker.0 00000100:00000001:3.0:1787829163.879390:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829163.879393:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829163.879395:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829163.879398:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829163.879401:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829163.879404:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829163.879406:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829163.879410:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829163.879412:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829163.879416:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829163.879418:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829163.879420:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829163.879427:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787829163.879429:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829163.879431:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1787829163.879432:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787829163.879434:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787829163.879435:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787829163.879437:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829163.879439:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787829163.879441:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829163.879456:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829163.879460:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d25a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646802432:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829163.879463:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1787829163.879465:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1787829163.879467:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829163.879468:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1787829163.879471:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829163.880249:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787829163.880253:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601343040 : -126234108208576 : ffff8d30d4d25a40) 00000100:00000001:3.0:1787829163.880260:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1787829163.880290:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:3.0:1787829163.880297:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787829163.880300:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829163.880302:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829163.880304:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1787829163.880309:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829163.880315:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298636823 00000100:00000001:3.0:1787829163.880319:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298638080 00000100:00000001:3.0:1787829163.880321:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298636823 00000100:00000001:3.0:1787829163.880324:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298638080 00000100:00000001:3.0:1787829163.880326:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829163.880328:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829163.880331:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1787829163.880337:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829163/real 1787829163] req@ffff8d30d4d25a40 x1874670646802432/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829188 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1787829163.880347:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787829163.880349:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829163.880353:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829163.880355:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829163.880356:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829163.880359:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829163.880360:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829163.880364:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d25a40 x1874670646802432/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829188 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829163.880371:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829163.880374:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829163.880377:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829163.880378:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829163.880380:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829163.880382:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829163.880384:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829163.880387:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829163.880391:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829163.880393:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829163.880397:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d25a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646802432:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829163.880403:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787829163.880405:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787829163.880406:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829163.880409:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787829163.880413:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829163.880415:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829163.880418:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d24f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646802560:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829163.880421:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1787829163.880424:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1787829163.880426:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829163.880428:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1787829163.880432:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829163.880436:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787829163.880438:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601340160 : -126234108211456 : ffff8d30d4d24f00) 00000100:00000001:3.0:1787829163.880442:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1787829163.880455:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:3.0:1787829163.880460:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787829163.880462:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829163.880464:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829163.880465:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1787829163.880468:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829163.880471:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298636823 00000100:00000001:3.0:1787829163.880473:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298638080 00000100:00000001:3.0:1787829163.880475:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298636823 00000100:00000001:3.0:1787829163.880477:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298638080 00000100:00000001:3.0:1787829163.880480:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829163.880482:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829163.880483:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1787829163.880487:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829163/real 1787829163] req@ffff8d30d4d24f00 x1874670646802560/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829188 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1787829163.880495:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787829163.880497:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829163.880499:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829163.880501:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829163.880502:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829163.880504:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829163.880505:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829163.880509:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d24f00 x1874670646802560/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829188 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829163.880515:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829163.880517:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829163.880519:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829163.880520:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829163.880521:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829163.880523:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829163.880525:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829163.880527:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829163.880530:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829163.880532:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829163.880535:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d24f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646802560:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829163.880538:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787829163.880540:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787829163.880541:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829163.880543:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787829163.880545:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829163.880547:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829163.880550:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d24000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646802688:192.168.203.143@tcp:8:kworker.0 00000100:00000001:3.0:1787829163.880553:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1787829163.880554:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1787829163.880556:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829163.880558:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1787829163.880561:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829163.880566:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787829163.880573:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601336320 : -126234108215296 : ffff8d30d4d24000) 00000100:00000001:3.0:1787829163.880576:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1787829163.880589:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:3.0:1787829163.880594:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787829163.880596:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829163.880597:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829163.880599:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1787829163.880628:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829163.880632:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298636837 00000100:00000001:3.0:1787829163.880634:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298638080 00000100:00000001:3.0:1787829163.880637:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298636837 00000100:00000001:3.0:1787829163.880640:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298638080 00000100:00000001:3.0:1787829163.880643:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829163.880645:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829163.880647:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1787829163.880652:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829163/real 1787829163] req@ffff8d30d4d24000 x1874670646802688/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829188 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1787829163.880660:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787829163.880661:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829163.880664:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829163.880666:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829163.880667:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829163.880669:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829163.880670:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829163.880674:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d24000 x1874670646802688/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829188 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829163.880682:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829163.880684:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829163.880687:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829163.880688:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829163.880690:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829163.880692:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829163.880694:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829163.880697:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829163.880700:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829163.880702:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829163.880705:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d24000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646802688:192.168.203.143@tcp:8:kworker.0 00000100:00000001:3.0:1787829163.880710:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829163.880712:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829163.880713:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829163.880715:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829163.880718:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829163.880720:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829163.880721:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829163.880724:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829163.880726:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829163.880729:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829163.880730:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829163.880732:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829163.880733:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829163.880735:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829163.880737:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829163.880738:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829163.880739:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829163.880741:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829163.880742:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829163.880745:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829163.880746:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829163.880748:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829163.880749:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829163.880750:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829163.880752:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829163.880753:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829163.880754:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829163.880756:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829163.880757:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829163.880759:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829163.880760:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829163.880762:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829163.880768:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787829163.880769:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829163.880771:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1787829163.880772:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829164.774687:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829164.774692:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829164.774742:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829164.774747:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829164.774751:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829165.798657:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829165.798661:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829165.798749:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829165.798752:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829165.798755:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829166.822712:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829166.822719:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829166.822781:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829166.822784:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829166.822787:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829167.846748:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829167.846759:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829167.846841:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829167.846847:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829167.846852:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829168.870719:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829168.870729:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829168.870824:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829168.870830:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829168.870835:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787829168.998811:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829168.998827:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1787829168.998832:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787829168.998837:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787829168.998844:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787829168.998856:0:40798:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 3969 00000100:00080000:2.0:1787829168.998873:0:40798:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787829168.998878:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829168.998882:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787829168.998890:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787829168.998895:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787829168.998901:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:2.0:1787829168.998905:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:2.0:1787829168.998910:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829168.998913:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:2.0:1787829168.998946:0:40798:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787829168.998953:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829168.998958:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787829168.998963:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787829168.998969:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829168.998975:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:2.0:1787829168.998996:0:40798:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829168.999009:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787829168.999011:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829168.999014:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787829168.999017:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787829168.999022:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829168.999034:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787829168.999046:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2f480 x1874670646802816/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787829168.999062:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787829168.999069:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829168.999110:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787829168.999115:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829168.999120:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1787829168.999122:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787829168.999124:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787829168.999129:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787829168.999134:0:40798:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3969 00000100:00080000:2.0:1787829168.999140:0:40798:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787829168.999144:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829168.999147:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787829168.999151:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:3.0:1787829168.999153:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:2.0:1787829168.999154:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787829168.999161:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:3.0:1787829168.999162:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000020:00000001:2.0:1787829168.999166:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:3.0:1787829168.999169:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787829168.999184:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787829168.999185:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829168.999190:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829168.999191:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 02000000:00000001:3.0:1787829168.999202:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000100:00080000:2.0:1787829168.999204:0:40798:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 02000000:00000001:3.0:1787829168.999210:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787829168.999211:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829168.999215:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787829168.999220:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829168.999225:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787829168.999233:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829168.999233:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646802816:192.168.203.143@tcp:250:kworker.0 00000100:00000001:2.0:1787829168.999246:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1787829168.999253:0:40798:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00000001:3.0:1787829168.999255:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000008:00000001:2.0:1787829168.999261:0:40798:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829168.999265:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:2.0:1787829168.999267:0:40798:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829168.999271:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829168.999277:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787829168.999278:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1787829168.999282:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829168.999288:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787829168.999289:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829168.999291:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787829168.999297:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829168.999315:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829168.999324:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00100000:2.0:1787829168.999326:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2da40 x1874670646802944/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787829168.999330:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417881728 : -126234291669888 : ffff8d30c9e2f480) 00000100:00000001:3.0:1787829168.999340:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:2.0:1787829168.999347:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787829168.999352:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829168.999357:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787829168.999361:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829168.999366:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1787829168.999368:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787829168.999370:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787829168.999373:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787829168.999376:0:40798:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3969 00000100:00080000:2.0:1787829168.999381:0:40798:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787829168.999383:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829168.999384:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787829168.999387:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787829168.999388:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787829168.999391:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000020:00000001:2.0:1787829168.999394:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:2.0:1787829168.999397:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829168.999400:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:2.0:1787829168.999426:0:40798:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000800:00000100:3.0:1787829168.999430:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:2.0:1787829168.999433:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829168.999439:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787829168.999445:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787829168.999450:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829168.999455:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:3.0:1787829168.999456:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000008:00100000:2.0:1787829168.999459:0:40798:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00000001:3.0:1787829168.999466:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000008:00000001:2.0:1787829168.999467:0:40798:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829168.999473:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829168.999475:0:40798:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829168.999484:0:2390:0:(events.c:68:request_out_callback()) Process leaving 02000000:00000001:2.0:1787829168.999491:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787829168.999495:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829168.999498:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829168.999499:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787829168.999503:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787829168.999509:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829168.999513:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4298643200 00000100:00000001:2.0:1787829168.999516:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787829168.999527:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2c000 x1874670646803072/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787829168.999532:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 00000100:00000001:2.0:1787829168.999538:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787829168.999543:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829168.999547:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787829168.999550:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829168.999554:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1787829168.999556:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787829168.999558:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787829168.999561:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:3.0:1787829168.999562:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4298643200 00000100:00080000:2.0:1787829168.999565:0:40798:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3969 00000100:00080000:2.0:1787829168.999575:0:40798:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787829168.999579:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829168.999580:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 00000100:00000001:2.0:1787829168.999582:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787829168.999588:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787829168.999590:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:3.0:1787829168.999591:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000020:00000001:2.0:1787829168.999594:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000020:00000001:2.0:1787829168.999601:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:2.0:1787829168.999603:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829168.999605:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:2.0:1787829168.999609:0:40798:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787829168.999612:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829168.999652:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829168.999658:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 02000000:00000001:2.0:1787829168.999665:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00100000:3.0:1787829168.999671:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829168/real 1787829168] req@ffff8d30c9e2f480 x1874670646802816/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829193 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1787829168.999673:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787829168.999682:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829168.999689:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1787829168.999694:0:40798:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000008:00000001:2.0:1787829168.999701:0:40798:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:3.0:1787829168.999704:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000001:2.0:1787829168.999705:0:40798:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:3.0:1787829168.999710:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 02000000:00000001:2.0:1787829168.999715:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:3.0:1787829168.999719:0:2390:0:(events.c:78:reply_in_callback()) Process entered 02000000:00000001:2.0:1787829168.999720:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829168.999734:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787829168.999736:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:3.0:1787829168.999739:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:2.0:1787829168.999740:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:3.0:1787829168.999746:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 02000000:00000001:2.0:1787829168.999747:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829168.999752:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829168.999757:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829168.999762:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2f480 x1874670646802816/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829193 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829168.999773:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2e940 x1874670646803200/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787829168.999776:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829168.999784:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829168.999793:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829168.999799:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829168.999801:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829168.999809:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829168.999814:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787829168.999817:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00080000:3.0:1787829168.999820:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00000001:2.0:1787829168.999825:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787829168.999831:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:2.0:1787829168.999836:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829168.999837:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829168.999850:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646802816:192.168.203.143@tcp:250:kworker.0 00000100:00000001:3.0:1787829168.999857:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829168.999861:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829168.999873:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829168.999876:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829168.999880:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829168.999883:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829168.999886:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829168.999892:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829168.999895:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829168.999899:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829168.999901:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829168.999903:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829168.999911:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787829168.999913:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829168.999916:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1787829168.999917:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787829168.999919:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787829168.999921:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787829168.999948:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829168.999953:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787829168.999956:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829168.999960:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829168.999966:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2da40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646802944:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829168.999982:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1787829168.999986:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1787829168.999989:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829168.999992:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1787829168.999997:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829169.000036:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787829169.000039:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417875008 : -126234291676608 : ffff8d30c9e2da40) 00000100:00000001:3.0:1787829169.000045:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1787829169.000065:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:3.0:1787829169.000070:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787829169.000073:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829169.000075:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829169.000076:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1787829169.000079:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829169.000083:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298641943 00000100:00000001:3.0:1787829169.000086:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298643200 00000100:00000001:3.0:1787829169.000089:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298641943 00000100:00000001:3.0:1787829169.000091:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298643200 00000100:00000001:3.0:1787829169.000093:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829169.000095:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829169.000097:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1787829169.000102:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829168/real 1787829169] req@ffff8d30c9e2da40 x1874670646802944/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829193 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1787829169.000110:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787829169.000112:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829169.000114:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829169.000115:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829169.000117:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829169.000118:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829169.000119:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829169.000123:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2da40 x1874670646802944/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829193 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829169.000129:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829169.000131:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829169.000134:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829169.000135:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829169.000136:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829169.000138:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829169.000140:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829169.000142:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829169.000145:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829169.000146:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829169.000149:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2da40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646802944:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829169.000152:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787829169.000154:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787829169.000155:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829169.000157:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787829169.000159:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829169.000161:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829169.000164:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646803072:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829169.000167:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1787829169.000169:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1787829169.000171:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829169.000172:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1787829169.000175:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829169.000179:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787829169.000180:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417868288 : -126234291683328 : ffff8d30c9e2c000) 00000100:00000001:3.0:1787829169.000183:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1787829169.000192:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:3.0:1787829169.000196:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787829169.000198:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829169.000200:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829169.000201:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1787829169.000203:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829169.000206:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298641943 00000100:00000001:3.0:1787829169.000208:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298643200 00000100:00000001:3.0:1787829169.000211:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298641943 00000100:00000001:3.0:1787829169.000213:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298643200 00000100:00000001:3.0:1787829169.000214:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829169.000216:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829169.000217:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1787829169.000221:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829168/real 1787829169] req@ffff8d30c9e2c000 x1874670646803072/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829193 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1787829169.000227:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787829169.000228:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829169.000232:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829169.000234:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829169.000235:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829169.000236:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829169.000238:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829169.000241:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2c000 x1874670646803072/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829193 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829169.000253:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829169.000255:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829169.000258:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829169.000259:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829169.000260:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829169.000262:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829169.000263:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829169.000265:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829169.000268:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829169.000269:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829169.000272:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646803072:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829169.000275:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787829169.000276:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787829169.000277:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829169.000279:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787829169.000280:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829169.000282:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829169.000284:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646803200:192.168.203.143@tcp:8:kworker.0 00000100:00000001:3.0:1787829169.000287:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1787829169.000288:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1787829169.000289:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829169.000291:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1787829169.000296:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829169.000300:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787829169.000302:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417878848 : -126234291672768 : ffff8d30c9e2e940) 00000100:00000001:3.0:1787829169.000304:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1787829169.000312:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:3.0:1787829169.000315:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787829169.000317:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829169.000319:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829169.000320:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1787829169.000322:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829169.000325:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298641956 00000100:00000001:3.0:1787829169.000327:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298643200 00000100:00000001:3.0:1787829169.000329:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298641956 00000100:00000001:3.0:1787829169.000331:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298643200 00000100:00000001:3.0:1787829169.000332:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829169.000334:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829169.000335:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1787829169.000338:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829168/real 1787829169] req@ffff8d30c9e2e940 x1874670646803200/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829193 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1787829169.000345:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787829169.000346:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829169.000348:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829169.000349:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829169.000351:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829169.000352:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829169.000353:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829169.000356:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2e940 x1874670646803200/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829193 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829169.000362:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829169.000364:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829169.000366:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829169.000367:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829169.000368:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829169.000369:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829169.000371:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829169.000373:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829169.000375:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829169.000377:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829169.000380:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646803200:192.168.203.143@tcp:8:kworker.0 00000100:00000001:3.0:1787829169.000383:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829169.000385:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829169.000386:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829169.000388:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829169.000390:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829169.000391:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829169.000392:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829169.000394:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829169.000395:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829169.000398:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829169.000399:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829169.000401:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829169.000402:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829169.000403:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829169.000405:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829169.000406:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829169.000407:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829169.000408:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829169.000409:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829169.000411:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829169.000412:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829169.000413:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829169.000414:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829169.000415:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829169.000416:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829169.000417:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829169.000418:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829169.000420:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829169.000420:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829169.000423:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829169.000425:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829169.000426:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829169.000429:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787829169.000431:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829169.000432:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1787829169.000434:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829169.894660:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829169.894665:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829169.894752:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829169.894755:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829169.894758:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829170.918752:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829170.918759:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829170.918866:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829170.918871:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829170.918875:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829171.942759:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829171.942766:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829171.942871:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829171.942877:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829171.942883:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829172.966696:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829172.966700:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829172.966752:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829172.966756:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829172.966759:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829173.990701:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829173.990710:0:815:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1787829173.990713:0:815:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1787829173.990716:0:815:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829173.990721:0:815:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1787829173.990723:0:815:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829173.990740:0:815:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829173.990742:0:815:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829173.990801:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829173.990808:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829173.990812:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787829174.118724:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829174.118735:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1787829174.118738:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787829174.118742:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787829174.118747:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787829174.118754:0:468:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 3974 00000100:00080000:0.0:1787829174.118767:0:468:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787829174.118769:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829174.118771:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787829174.118775:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787829174.118778:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787829174.118781:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:0.0:1787829174.118783:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:0.0:1787829174.118785:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829174.118787:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:0.0:1787829174.118807:0:468:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787829174.118810:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829174.118812:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787829174.118816:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787829174.118818:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829174.118821:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1787829174.118824:0:468:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829174.118839:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787829174.118842:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829174.118846:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787829174.118849:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787829174.118854:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829174.118871:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787829174.118882:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d24000 x1874670646803328/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787829174.118911:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787829174.118917:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829174.118933:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787829174.118936:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829174.118939:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1787829174.118941:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787829174.118943:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787829174.118945:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787829174.118948:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3974 00000100:00080000:0.0:1787829174.118952:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787829174.118953:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829174.118955:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787829174.118957:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787829174.118959:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787829174.118961:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000020:00000001:0.0:1787829174.118964:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:0.0:1787829174.118966:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829174.118968:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:0.0:1787829174.118971:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787829174.118974:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829174.118976:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:3.0:1787829174.118981:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:0.0:1787829174.118983:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787829174.118987:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829174.118988:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787829174.118989:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:3.0:1787829174.118993:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000008:00100000:0.0:1787829174.118996:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00000001:3.0:1787829174.118997:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787829174.119001:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:0.0:1787829174.119002:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829174.119007:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829174.119009:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787829174.119012:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787829174.119013:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787829174.119016:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829174.119019:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829174.119020:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787829174.119024:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787829174.119028:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829174.119028:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d24000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646803328:192.168.203.143@tcp:250:kworker.0 00000100:00000001:3.0:1787829174.119036:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:0.0:1787829174.119038:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829174.119042:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000100:00100000:0.0:1787829174.119048:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d24f00 x1874670646803456/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:3.0:1787829174.119048:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829174.119055:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:0.0:1787829174.119063:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 02000000:00000001:3.0:1787829174.119063:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829174.119066:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829174.119071:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787829174.119074:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829174.119077:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1787829174.119078:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787829174.119080:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787829174.119082:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787829174.119084:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3974 00000100:00000001:3.0:1787829174.119086:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00080000:0.0:1787829174.119087:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787829174.119091:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601336320 : -126234108215296 : ffff8d30d4d24000) 00000100:00000001:0.0:1787829174.119092:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829174.119095:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:3.0:1787829174.119098:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000020:00000001:0.0:1787829174.119099:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787829174.119102:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787829174.119104:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000020:00000001:0.0:1787829174.119106:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:0.0:1787829174.119108:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829174.119109:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:0.0:1787829174.119112:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787829174.119114:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829174.119116:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787829174.119118:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787829174.119120:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829174.119122:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1787829174.119123:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1787829174.119125:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829174.119126:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829174.119131:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787829174.119133:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829174.119134:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787829174.119135:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787829174.119136:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829174.119141:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787829174.119146:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d25a40 x1874670646803584/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1787829174.119151:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:0.0:1787829174.119160:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787829174.119162:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829174.119164:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787829174.119167:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829174.119170:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1787829174.119172:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1787829174.119173:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787829174.119178:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787829174.119180:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829174.119185:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829174.119186:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:3.0:1787829174.119188:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00080000:0.0:1787829174.119194:0:468:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3974 00000100:00000001:3.0:1787829174.119196:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787829174.119204:0:468:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787829174.119205:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4298648320 00000100:00000001:0.0:1787829174.119212:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829174.119214:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787829174.119218:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:3.0:1787829174.119220:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 00000020:00000001:0.0:1787829174.119221:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787829174.119227:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:3.0:1787829174.119232:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4298648320 00000020:00000001:0.0:1787829174.119233:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:0.0:1787829174.119241:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829174.119242:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1787829174.119247:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:3.0:1787829174.119249:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829174.119255:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787829174.119259:0:468:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:3.0:1787829174.119261:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00000001:0.0:1787829174.119268:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829174.119270:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829174/real 1787829174] req@ffff8d30d4d24000 x1874670646803328/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829199 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:0.0:1787829174.119274:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787829174.119280:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787829174.119286:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829174.119289:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000400:00000001:3.0:1787829174.119289:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000008:00100000:0.0:1787829174.119296:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000400:00000001:3.0:1787829174.119296:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000008:00000001:0.0:1787829174.119301:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829174.119304:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829174.119305:0:2390:0:(events.c:78:reply_in_callback()) Process entered 02000000:00000001:0.0:1787829174.119309:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:3.0:1787829174.119311:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 02000000:00000001:0.0:1787829174.119313:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829174.119317:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:3.0:1787829174.119318:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:0.0:1787829174.119319:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787829174.119322:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829174.119327:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787829174.119329:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829174.119332:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787829174.119338:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d270c0 x1874670646803712/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787829174.119347:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d24000 x1874670646803328/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829199 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787829174.119349:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787829174.119353:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829174.119358:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829174.119368:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829174.119372:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829174.119375:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829174.119377:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829174.119379:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829174.119382:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829174.119384:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829174.119387:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829174.119391:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829174.119393:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829174.119396:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d24000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646803328:192.168.203.143@tcp:250:kworker.0 00000100:00000001:3.0:1787829174.119401:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829174.119403:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829174.119405:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829174.119407:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829174.119411:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829174.119413:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829174.119415:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829174.119420:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829174.119421:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829174.119424:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829174.119426:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829174.119428:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829174.119438:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787829174.119441:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829174.119444:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1787829174.119446:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787829174.119448:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787829174.119450:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787829174.119451:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829174.119455:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787829174.119457:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829174.119460:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829174.119465:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d24f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646803456:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829174.119470:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1787829174.119472:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1787829174.119475:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829174.119477:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1787829174.119481:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829174.119505:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787829174.119507:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601340160 : -126234108211456 : ffff8d30d4d24f00) 00000100:00000001:3.0:1787829174.119512:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1787829174.119524:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:3.0:1787829174.119528:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787829174.119530:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829174.119531:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829174.119532:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1787829174.119535:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829174.119539:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298647062 00000100:00000001:3.0:1787829174.119541:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298648320 00000100:00000001:3.0:1787829174.119543:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298647062 00000100:00000001:3.0:1787829174.119545:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298648320 00000100:00000001:3.0:1787829174.119546:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829174.119548:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829174.119549:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1787829174.119552:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829174/real 1787829174] req@ffff8d30d4d24f00 x1874670646803456/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829199 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1787829174.119578:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787829174.119579:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829174.119581:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829174.119583:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829174.119584:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829174.119585:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829174.119587:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829174.119590:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d24f00 x1874670646803456/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829199 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829174.119596:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829174.119598:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829174.119600:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829174.119601:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829174.119602:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829174.119603:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829174.119605:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829174.119607:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829174.119609:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829174.119611:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829174.119613:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d24f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646803456:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829174.119616:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787829174.119617:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787829174.119619:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829174.119620:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787829174.119622:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829174.119624:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829174.119626:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d25a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646803584:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829174.119680:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1787829174.119684:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1787829174.119687:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829174.119690:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1787829174.119710:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829174.119715:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787829174.119717:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601343040 : -126234108208576 : ffff8d30d4d25a40) 00000100:00000001:3.0:1787829174.119721:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1787829174.119733:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:3.0:1787829174.119738:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787829174.119740:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829174.119742:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829174.119743:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1787829174.119745:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829174.119748:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298647063 00000100:00000001:3.0:1787829174.119751:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298648320 00000100:00000001:3.0:1787829174.119753:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298647063 00000100:00000001:3.0:1787829174.119755:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298648320 00000100:00000001:3.0:1787829174.119757:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829174.119758:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829174.119760:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1787829174.119763:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829174/real 1787829174] req@ffff8d30d4d25a40 x1874670646803584/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829199 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1787829174.119769:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787829174.119770:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829174.119772:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829174.119773:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829174.119775:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829174.119776:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829174.119778:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829174.119781:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d25a40 x1874670646803584/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829199 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829174.119787:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829174.119788:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829174.119791:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829174.119792:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829174.119793:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829174.119795:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829174.119796:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829174.119799:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829174.119801:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829174.119802:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829174.119805:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d25a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646803584:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829174.119808:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787829174.119809:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787829174.119810:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829174.119811:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787829174.119813:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829174.119814:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829174.119817:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d270c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646803712:192.168.203.143@tcp:8:kworker.0 00000100:00000001:3.0:1787829174.119819:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1787829174.119820:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1787829174.119822:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829174.119823:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1787829174.119826:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829174.119830:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787829174.119831:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601348800 : -126234108202816 : ffff8d30d4d270c0) 00000100:00000001:3.0:1787829174.119834:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1787829174.119842:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:3.0:1787829174.119845:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787829174.119846:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829174.119848:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829174.119849:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1787829174.119851:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829174.119854:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298647076 00000100:00000001:3.0:1787829174.119856:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298648320 00000100:00000001:3.0:1787829174.119858:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298647076 00000100:00000001:3.0:1787829174.119860:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298648320 00000100:00000001:3.0:1787829174.119862:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829174.119864:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829174.119865:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1787829174.119868:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829174/real 1787829174] req@ffff8d30d4d270c0 x1874670646803712/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829199 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1787829174.119874:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787829174.119875:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829174.119876:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829174.119877:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829174.119878:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829174.119880:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829174.119881:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829174.119884:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d270c0 x1874670646803712/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829199 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829174.119907:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829174.119909:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829174.119911:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829174.119913:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829174.119914:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829174.119916:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829174.119917:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829174.119920:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829174.119922:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829174.119924:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829174.119926:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d270c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646803712:192.168.203.143@tcp:8:kworker.0 00000100:00000001:3.0:1787829174.119929:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829174.119931:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829174.119933:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829174.119934:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829174.119937:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829174.119938:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829174.119940:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829174.119942:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829174.119943:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829174.119946:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829174.119947:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829174.119949:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829174.119950:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829174.119951:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829174.119953:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829174.119954:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829174.119955:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829174.119956:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829174.119957:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829174.119958:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829174.119959:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829174.119960:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829174.119961:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829174.119962:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829174.119963:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829174.119964:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829174.119965:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829174.119966:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829174.119967:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829174.119969:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829174.119970:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829174.119972:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829174.119975:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787829174.119976:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829174.119977:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1787829174.119979:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829175.014725:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829175.014734:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829175.014806:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829175.014809:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829175.014812:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829176.038731:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829176.038738:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829176.038905:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829176.038909:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829176.038913:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829177.062830:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829177.062838:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829177.062908:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829177.062913:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829177.062917:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829178.086669:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829178.086674:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829178.086730:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829178.086750:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829178.086753:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829179.110706:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829179.110714:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829179.110775:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829179.110779:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829179.110781:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787829179.238688:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829179.238696:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1787829179.238698:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787829179.238701:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787829179.238705:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787829179.238711:0:468:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 3979 00000100:00080000:0.0:1787829179.238719:0:468:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787829179.238721:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829179.238723:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787829179.238726:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787829179.238728:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787829179.238731:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:0.0:1787829179.238733:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:0.0:1787829179.238734:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829179.238735:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:0.0:1787829179.238738:0:468:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787829179.238740:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829179.238742:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787829179.238745:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787829179.238746:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829179.238748:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1787829179.238750:0:468:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829179.238760:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787829179.238761:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829179.238764:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787829179.238766:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787829179.238769:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829179.238778:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787829179.238786:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d27c00 x1874670646803840/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787829179.238793:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787829179.238797:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829179.238809:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787829179.238811:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829179.238814:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1787829179.238815:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787829179.238816:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787829179.238818:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787829179.238820:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3979 00000100:00080000:0.0:1787829179.238822:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787829179.238824:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829179.238825:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787829179.238826:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787829179.238827:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787829179.238829:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000020:00000001:0.0:1787829179.238831:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:0.0:1787829179.238833:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829179.238839:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:0.0:1787829179.238842:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787829179.238844:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829179.238845:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:3.0:1787829179.238849:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:0.0:1787829179.238850:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829179.238863:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787829179.238867:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 02000000:00000001:0.0:1787829179.238869:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829179.238870:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787829179.238873:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:3.0:1787829179.238874:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000008:00100000:0.0:1787829179.238880:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 02000000:00000001:3.0:1787829179.238880:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787829179.238883:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000008:00000001:0.0:1787829179.238885:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829179.238889:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829179.238890:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829179.238896:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00100000:3.0:1787829179.238897:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d27c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646803840:192.168.203.143@tcp:250:kworker.0 02000000:00000001:0.0:1787829179.238900:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829179.238903:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:0.0:1787829179.238905:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787829179.238907:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787829179.238909:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787829179.238910:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829179.238914:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1787829179.238915:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829179.238918:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829179.238925:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829179.238933:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00100000:0.0:1787829179.238936:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d252c0 x1874670646803968/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787829179.238936:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601351680 : -126234108199936 : ffff8d30d4d27c00) 00000100:00000001:3.0:1787829179.238941:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:0.0:1787829179.238951:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787829179.238953:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829179.238956:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787829179.238958:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829179.238961:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1787829179.238963:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787829179.238964:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787829179.238966:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787829179.238968:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3979 00000100:00080000:0.0:1787829179.238971:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787829179.238974:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829179.238977:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000800:00000100:3.0:1787829179.238977:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000020:00000001:0.0:1787829179.238982:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787829179.238983:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787829179.238985:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000020:00000001:0.0:1787829179.238987:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:0.0:1787829179.238988:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829179.238990:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787829179.238994:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829179.238996:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829179.238998:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787829179.238999:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:3.0:1787829179.239004:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787829179.239005:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787829179.239011:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829179.239013:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:3.0:1787829179.239013:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4298653440 02000000:00000001:0.0:1787829179.239018:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787829179.239021:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829179.239022:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1787829179.239024:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1787829179.239027:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829179.239027:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1787829179.239031:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829179.239036:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787829179.239038:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829179.239038:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4298653440 00000100:00000001:0.0:1787829179.239042:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787829179.239044:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787829179.239046:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829179.239048:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1787829179.239051:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829179.239054:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00100000:0.0:1787829179.239058:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d25680 x1874670646804096/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787829179.239060:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829179.239082:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00000001:0.0:1787829179.239086:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787829179.239091:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829179.239091:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829179/real 1787829179] req@ffff8d30d4d27c00 x1874670646803840/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829204 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787829179.239096:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787829179.239101:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829179.239109:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1787829179.239113:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000400:00000001:3.0:1787829179.239115:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787829179.239118:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00080000:0.0:1787829179.239119:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787829179.239122:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787829179.239124:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787829179.239129:0:468:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3979 00000100:00000001:3.0:1787829179.239129:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829179.239134:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:0.0:1787829179.239135:0:468:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787829179.239141:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829179.239141:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787829179.239144:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:3.0:1787829179.239146:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1787829179.239147:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787829179.239151:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787829179.239153:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000020:00000001:0.0:1787829179.239156:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:0.0:1787829179.239158:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00100000:3.0:1787829179.239159:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d27c00 x1874670646803840/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829204 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787829179.239160:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:0.0:1787829179.239166:0:468:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787829179.239170:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829179.239174:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787829179.239177:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787829179.239181:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829179.239181:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829179.239182:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1787829179.239187:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00000001:3.0:1787829179.239187:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000008:00000001:0.0:1787829179.239190:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829179.239192:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829179.239194:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829179.239196:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 02000000:00000001:0.0:1787829179.239199:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:3.0:1787829179.239199:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829179.239202:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829179.239203:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829179.239205:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 02000000:00000001:0.0:1787829179.239207:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829179.239210:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00080000:3.0:1787829179.239211:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 02000000:00000001:0.0:1787829179.239212:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:3.0:1787829179.239215:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 02000000:00000001:0.0:1787829179.239216:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829179.239223:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d27c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646803840:192.168.203.143@tcp:250:kworker.0 00000100:00000001:0.0:1787829179.239224:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829179.239231:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787829179.239235:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d26940 x1874670646804224/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787829179.239236:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829179.239239:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829179.239243:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 00000100:00000001:0.0:1787829179.239247:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 02000000:00000001:3.0:1787829179.239247:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00000001:0.0:1787829179.239251:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829179.239252:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829179.239255:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 00000100:00000001:0.0:1787829179.239256:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829179.239260:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829179.239262:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829179.239266:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829179.239268:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829179.239269:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829179.239276:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787829179.239278:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829179.239280:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1787829179.239281:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787829179.239283:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787829179.239284:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787829179.239285:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829179.239287:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787829179.239288:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829179.239290:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829179.239293:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d252c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646803968:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829179.239295:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1787829179.239297:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1787829179.239298:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829179.239299:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1787829179.239302:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829179.239306:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787829179.239307:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601341120 : -126234108210496 : ffff8d30d4d252c0) 00000100:00000001:3.0:1787829179.239311:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1787829179.239337:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:3.0:1787829179.239341:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787829179.239343:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829179.239344:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829179.239345:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1787829179.239347:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829179.239351:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298652182 00000100:00000001:3.0:1787829179.239353:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298653440 00000100:00000001:3.0:1787829179.239354:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298652182 00000100:00000001:3.0:1787829179.239356:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298653440 00000100:00000001:3.0:1787829179.239357:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829179.239358:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829179.239360:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1787829179.239363:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829179/real 1787829179] req@ffff8d30d4d252c0 x1874670646803968/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829204 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1787829179.239382:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787829179.239385:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829179.239403:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829179.239404:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829179.239405:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829179.239407:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829179.239408:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829179.239411:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d252c0 x1874670646803968/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829204 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829179.239416:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829179.239418:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829179.239420:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829179.239421:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829179.239422:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829179.239423:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829179.239425:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829179.239426:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829179.239428:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829179.239430:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829179.239432:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d252c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646803968:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829179.239434:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787829179.239435:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787829179.239436:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829179.239438:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787829179.239439:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829179.239441:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829179.239443:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d25680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646804096:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829179.239445:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1787829179.239446:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1787829179.239448:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829179.239449:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1787829179.239467:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829179.239470:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787829179.239471:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601342080 : -126234108209536 : ffff8d30d4d25680) 00000100:00000001:3.0:1787829179.239474:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1787829179.239482:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:3.0:1787829179.239484:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787829179.239486:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829179.239487:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829179.239488:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1787829179.239489:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829179.239492:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298652182 00000100:00000001:3.0:1787829179.239493:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298653440 00000100:00000001:3.0:1787829179.239495:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298652182 00000100:00000001:3.0:1787829179.239496:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298653440 00000100:00000001:3.0:1787829179.239498:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829179.239499:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829179.239500:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1787829179.239503:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829179/real 1787829179] req@ffff8d30d4d25680 x1874670646804096/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829204 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1787829179.239507:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787829179.239508:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829179.239509:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829179.239510:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829179.239511:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829179.239512:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829179.239513:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829179.239515:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d25680 x1874670646804096/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829204 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829179.239519:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829179.239521:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829179.239522:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829179.239523:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829179.239524:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829179.239526:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829179.239527:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829179.239528:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829179.239530:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829179.239532:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829179.239533:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d25680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646804096:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829179.239536:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787829179.239536:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787829179.239537:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829179.239538:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787829179.239539:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829179.239541:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829179.239542:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d26940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646804224:192.168.203.143@tcp:8:kworker.0 00000100:00000001:3.0:1787829179.239544:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1787829179.239545:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1787829179.239546:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829179.239547:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1787829179.239549:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829179.239552:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787829179.239553:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601346880 : -126234108204736 : ffff8d30d4d26940) 00000100:00000001:3.0:1787829179.239555:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1787829179.239560:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:3.0:1787829179.239563:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787829179.239564:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829179.239565:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829179.239566:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1787829179.239568:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829179.239570:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298652195 00000100:00000001:3.0:1787829179.239571:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298653440 00000100:00000001:3.0:1787829179.239573:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298652195 00000100:00000001:3.0:1787829179.239575:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298653440 00000100:00000001:3.0:1787829179.239576:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829179.239578:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829179.239579:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1787829179.239581:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829179/real 1787829179] req@ffff8d30d4d26940 x1874670646804224/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829204 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1787829179.239585:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787829179.239586:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829179.239587:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829179.239588:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829179.239589:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829179.239590:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829179.239591:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829179.239593:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d26940 x1874670646804224/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829204 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829179.239597:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829179.239598:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829179.239600:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829179.239601:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829179.239602:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829179.239603:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829179.239604:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829179.239605:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829179.239607:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829179.239608:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829179.239610:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d26940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646804224:192.168.203.143@tcp:8:kworker.0 00000100:00000001:3.0:1787829179.239612:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829179.239613:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829179.239614:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829179.239615:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829179.239617:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829179.239618:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829179.239619:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829179.239620:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829179.239621:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829179.239623:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829179.239624:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829179.239625:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829179.239625:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829179.239626:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829179.239656:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829179.239658:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829179.239659:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829179.239661:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829179.239663:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829179.239665:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829179.239666:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829179.239668:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829179.239669:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829179.239670:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829179.239672:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829179.239673:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829179.239675:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829179.239676:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829179.239677:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829179.239680:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829179.239681:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829179.239683:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829179.239687:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787829179.239688:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829179.239690:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1787829179.239692:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829180.134666:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829180.134671:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829180.134717:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829180.134721:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829180.134724:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829181.158704:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829181.158709:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829181.158799:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829181.158802:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829181.158805:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829182.182746:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829182.182756:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829182.182853:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829182.182859:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829182.182863:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829183.206699:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829183.206704:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829183.206736:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829183.206740:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829183.206743:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829184.230729:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829184.230738:0:815:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1787829184.230741:0:815:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1787829184.230744:0:815:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829184.230749:0:815:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1787829184.230752:0:815:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829184.230754:0:815:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829184.230757:0:815:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829184.230837:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829184.230841:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829184.230845:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787829184.358687:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829184.358695:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1787829184.358697:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787829184.358701:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787829184.358704:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787829184.358711:0:468:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 3984 00000100:00080000:0.0:1787829184.358720:0:468:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787829184.358722:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829184.358724:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787829184.358728:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787829184.358730:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787829184.358743:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:0.0:1787829184.358746:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:0.0:1787829184.358748:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829184.358749:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:0.0:1787829184.358753:0:468:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787829184.358755:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829184.358758:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787829184.358761:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787829184.358763:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829184.358766:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1787829184.358768:0:468:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829184.358777:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787829184.358778:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829184.358781:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787829184.358783:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787829184.358786:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829184.358796:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787829184.358827:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d26940 x1874670646804352/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787829184.358836:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787829184.358840:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829184.358852:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787829184.358854:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829184.358857:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1787829184.358858:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787829184.358860:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787829184.358862:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787829184.358864:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3984 00000100:00080000:0.0:1787829184.358877:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787829184.358878:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829184.358880:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787829184.358882:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787829184.358883:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787829184.358885:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000020:00000001:0.0:1787829184.358886:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:0.0:1787829184.358888:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829184.358888:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787829184.358890:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:0.0:1787829184.358895:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:3.0:1787829184.358895:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787829184.358899:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829184.358900:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 02000000:00000001:0.0:1787829184.358902:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:3.0:1787829184.358907:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 02000000:00000001:0.0:1787829184.358909:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829184.358911:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829184.358912:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829184.358917:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 02000000:00000001:3.0:1787829184.358919:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000008:00100000:0.0:1787829184.358921:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 02000000:00000001:3.0:1787829184.358923:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000008:00000001:0.0:1787829184.358925:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829184.358927:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829184.358928:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829184.358935:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00100000:3.0:1787829184.358935:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d26940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646804352:192.168.203.143@tcp:250:kworker.0 02000000:00000001:0.0:1787829184.358938:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829184.358943:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:3.0:1787829184.358944:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787829184.358946:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787829184.358949:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829184.358951:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:0.0:1787829184.358954:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829184.358956:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787829184.358961:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d25680 x1874670646804480/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:3.0:1787829184.358962:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1787829184.358970:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829184.358978:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787829184.358980:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829184.358984:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787829184.358986:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829184.358988:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1787829184.358990:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:3.0:1787829184.358991:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00080000:0.0:1787829184.358992:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787829184.358995:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601346880 : -126234108204736 : ffff8d30d4d26940) 00000100:00000001:0.0:1787829184.358996:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787829184.359000:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3984 00000100:00080000:0.0:1787829184.359003:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787829184.359005:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829184.359006:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787829184.359007:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787829184.359008:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787829184.359009:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000020:00000001:0.0:1787829184.359010:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:0.0:1787829184.359011:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829184.359012:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:0.0:1787829184.359014:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787829184.359015:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829184.359016:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787829184.359018:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787829184.359019:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829184.359020:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1787829184.359022:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1787829184.359023:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829184.359024:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829184.359027:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787829184.359027:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829184.359029:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787829184.359030:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787829184.359031:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829184.359033:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787829184.359036:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d252c0 x1874670646804608/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787829184.359041:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787829184.359042:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829184.359044:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787829184.359046:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829184.359047:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1787829184.359048:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787829184.359049:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787829184.359051:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787829184.359053:0:468:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3984 00000100:00080000:0.0:1787829184.359054:0:468:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787829184.359055:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829184.359056:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787829184.359058:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787829184.359058:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787829184.359060:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000020:00000001:0.0:1787829184.359061:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:0.0:1787829184.359062:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829184.359063:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:0.0:1787829184.359065:0:468:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:3.0:1787829184.359069:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:0.0:1787829184.359075:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829184.359077:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787829184.359079:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787829184.359080:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829184.359081:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1787829184.359083:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1787829184.359085:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829184.359087:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829184.359090:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787829184.359092:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829184.359093:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787829184.359095:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787829184.359096:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829184.359100:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787829184.359103:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d27c00 x1874670646804736/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1787829184.359107:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:0.0:1787829184.359114:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787829184.359116:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829184.359119:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829184.359120:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787829184.359125:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829184.359127:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829184.359129:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1787829184.359133:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829184.359136:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4298658560 00000100:00000001:3.0:1787829184.359143:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 00000100:00000001:3.0:1787829184.359147:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4298658560 00000100:00000001:3.0:1787829184.359150:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 00000100:00000001:3.0:1787829184.359153:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829184.359163:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829184.359166:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1787829184.359174:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829184/real 1787829184] req@ffff8d30d4d26940 x1874670646804352/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829209 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1787829184.359186:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787829184.359188:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829184.359321:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829184.359323:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829184.359325:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829184.359327:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829184.359329:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829184.359334:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d26940 x1874670646804352/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829209 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829184.359341:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829184.359344:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829184.359381:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829184.359383:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829184.359384:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829184.359386:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829184.359388:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829184.359391:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829184.359395:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829184.359398:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829184.359401:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d26940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646804352:192.168.203.143@tcp:250:kworker.0 00000100:00000001:3.0:1787829184.359406:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829184.359408:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829184.359410:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829184.359412:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829184.359415:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829184.359417:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829184.359419:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829184.359421:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829184.359423:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829184.359425:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829184.359426:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829184.359427:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829184.359433:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787829184.359435:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829184.359436:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1787829184.359437:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787829184.359439:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787829184.359440:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787829184.359441:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829184.359443:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787829184.359444:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829184.359445:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829184.359447:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d25680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646804480:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829184.359450:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1787829184.359451:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1787829184.359452:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829184.359454:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1787829184.359456:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829184.359460:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787829184.359461:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601342080 : -126234108209536 : ffff8d30d4d25680) 00000100:00000001:3.0:1787829184.359465:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1787829184.359475:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:3.0:1787829184.359479:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787829184.359480:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829184.359481:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829184.359483:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1787829184.359484:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829184.359487:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298657302 00000100:00000001:3.0:1787829184.359490:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298658560 00000100:00000001:3.0:1787829184.359491:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298657302 00000100:00000001:3.0:1787829184.359493:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298658560 00000100:00000001:3.0:1787829184.359494:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829184.359495:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829184.359496:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1787829184.359499:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829184/real 1787829184] req@ffff8d30d4d25680 x1874670646804480/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829209 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1787829184.359505:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787829184.359506:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829184.359507:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829184.359509:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829184.359510:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829184.359511:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829184.359512:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829184.359514:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d25680 x1874670646804480/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829209 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829184.359519:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829184.359520:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829184.359522:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829184.359523:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829184.359524:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829184.359525:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829184.359526:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829184.359527:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829184.359536:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829184.359538:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829184.359540:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d25680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646804480:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829184.359542:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787829184.359543:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787829184.359544:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829184.359546:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787829184.359547:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829184.359548:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829184.359550:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d252c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646804608:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829184.359552:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1787829184.359554:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1787829184.359555:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829184.359556:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1787829184.359558:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829184.359561:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787829184.359561:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601341120 : -126234108210496 : ffff8d30d4d252c0) 00000100:00000001:3.0:1787829184.359564:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1787829184.359571:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:3.0:1787829184.359574:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787829184.359575:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829184.359576:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829184.359577:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1787829184.359579:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829184.359581:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298657302 00000100:00000001:3.0:1787829184.359583:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298658560 00000100:00000001:3.0:1787829184.359584:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298657302 00000100:00000001:3.0:1787829184.359586:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298658560 00000100:00000001:3.0:1787829184.359587:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829184.359588:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829184.359589:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1787829184.359592:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829184/real 1787829184] req@ffff8d30d4d252c0 x1874670646804608/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829209 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1787829184.359625:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787829184.359626:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829184.359628:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829184.359630:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829184.359631:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829184.359633:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829184.359634:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829184.359637:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d252c0 x1874670646804608/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829209 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829184.359643:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829184.359644:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829184.359646:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829184.359647:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829184.359648:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829184.359649:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829184.359651:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829184.359653:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829184.359656:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829184.359659:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829184.359662:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d252c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646804608:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829184.359667:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787829184.359668:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787829184.359669:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829184.359671:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787829184.359673:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829184.359676:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829184.359680:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d27c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646804736:192.168.203.143@tcp:8:kworker.0 00000100:00000001:3.0:1787829184.359684:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1787829184.359686:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1787829184.359687:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829184.359689:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1787829184.359692:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829184.359696:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787829184.359698:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601351680 : -126234108199936 : ffff8d30d4d27c00) 00000100:00000001:3.0:1787829184.359701:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1787829184.359714:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:3.0:1787829184.359718:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787829184.359721:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829184.359722:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829184.359724:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1787829184.359727:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829184.359730:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298657316 00000100:00000001:3.0:1787829184.359732:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298658560 00000100:00000001:3.0:1787829184.359734:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298657316 00000100:00000001:3.0:1787829184.359735:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298658560 00000100:00000001:3.0:1787829184.359737:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829184.359738:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829184.359739:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1787829184.359743:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829184/real 1787829184] req@ffff8d30d4d27c00 x1874670646804736/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829209 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1787829184.359749:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787829184.359750:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829184.359751:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829184.359752:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829184.359753:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829184.359755:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829184.359756:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829184.359758:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d27c00 x1874670646804736/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829209 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829184.359763:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829184.359764:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829184.359766:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829184.359766:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829184.359767:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829184.359768:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829184.359769:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829184.359771:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829184.359773:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829184.359774:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829184.359776:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d27c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646804736:192.168.203.143@tcp:8:kworker.0 00000100:00000001:3.0:1787829184.359778:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829184.359779:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829184.359780:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829184.359781:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829184.359783:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829184.359784:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829184.359785:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829184.359787:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829184.359787:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829184.359789:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829184.359790:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829184.359791:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829184.359792:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829184.359793:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829184.359794:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829184.359795:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829184.359796:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829184.359797:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829184.359798:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829184.359799:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829184.359799:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829184.359800:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829184.359801:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829184.359802:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829184.359814:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829184.359814:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829184.359828:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829184.359830:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829184.359830:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829184.359833:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829184.359833:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829184.359835:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829184.359837:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787829184.359838:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829184.359839:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1787829184.359840:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829185.254666:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829185.254672:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829185.254729:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829185.254734:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829185.254738:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829186.278685:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829186.278692:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829186.278784:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829186.278787:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829186.278790:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829187.302664:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829187.302671:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829187.302723:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829187.302726:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829187.302729:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829188.326707:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829188.326712:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829188.326763:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829188.326767:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829188.326769:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829189.350717:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829189.350724:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829189.350780:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829189.350794:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829189.350797:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787829189.478696:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829189.478704:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1787829189.478706:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787829189.478709:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787829189.478713:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787829189.478718:0:468:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 3989 00000100:00080000:0.0:1787829189.478727:0:468:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787829189.478729:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829189.478731:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787829189.478734:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787829189.478736:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787829189.478739:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:0.0:1787829189.478740:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:0.0:1787829189.478742:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829189.478743:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:0.0:1787829189.478746:0:468:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787829189.478748:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829189.478750:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787829189.478753:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787829189.478755:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829189.478756:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1787829189.478758:0:468:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829189.478768:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787829189.478769:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829189.478772:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787829189.478774:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787829189.478777:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829189.478799:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787829189.478806:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d270c0 x1874670646804864/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787829189.478814:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787829189.478817:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829189.478828:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787829189.478830:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829189.478833:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1787829189.478834:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787829189.478835:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787829189.478837:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787829189.478839:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3989 00000100:00080000:0.0:1787829189.478841:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787829189.478843:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829189.478844:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787829189.478846:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787829189.478847:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787829189.478848:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000020:00000001:0.0:1787829189.478850:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:0.0:1787829189.478852:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829189.478853:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:0.0:1787829189.478856:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787829189.478858:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829189.478860:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:3.0:1787829189.478862:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:0.0:1787829189.478865:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829189.478867:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 02000000:00000001:0.0:1787829189.478868:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829189.478871:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1787829189.478872:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:3.0:1787829189.478876:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000008:00100000:0.0:1787829189.478878:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00000001:3.0:1787829189.478879:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:0.0:1787829189.478883:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829189.478886:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829189.478886:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787829189.478891:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787829189.478892:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787829189.478895:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829189.478896:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829189.478898:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787829189.478900:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787829189.478903:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829189.478904:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d270c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646804864:192.168.203.143@tcp:250:kworker.0 00000100:00000001:0.0:1787829189.478907:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829189.478909:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00100000:0.0:1787829189.478914:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d25a40 x1874670646804992/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:3.0:1787829189.478914:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1787829189.478918:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829189.478923:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:0.0:1787829189.478927:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 02000000:00000001:3.0:1787829189.478929:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829189.478930:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829189.478936:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787829189.478939:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829189.478941:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1787829189.478943:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787829189.478944:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787829189.478946:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:3.0:1787829189.478947:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00080000:0.0:1787829189.478948:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3989 00000100:00000001:3.0:1787829189.478952:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601348800 : -126234108202816 : ffff8d30d4d270c0) 00000100:00080000:0.0:1787829189.478953:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787829189.478957:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829189.478957:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:0.0:1787829189.478958:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787829189.478961:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787829189.478962:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787829189.478964:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000020:00000001:0.0:1787829189.478965:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:0.0:1787829189.478967:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829189.478968:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:0.0:1787829189.478971:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787829189.478972:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829189.478974:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787829189.478976:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787829189.478977:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829189.478978:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1787829189.478979:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1787829189.478981:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829189.478983:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829189.478986:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787829189.478987:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829189.478988:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787829189.478989:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787829189.478990:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:3.0:1787829189.478994:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:0.0:1787829189.478996:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787829189.479001:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d24f00 x1874670646805120/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787829189.479005:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1787829189.479009:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787829189.479011:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829189.479014:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829189.479015:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787829189.479018:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829189.479021:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829189.479022:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787829189.479028:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1787829189.479031:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4298663680 00000100:00080000:0.0:1787829189.479034:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1787829189.479037:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787829189.479038:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787829189.479044:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1787829189.479045:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787829189.479051:0:468:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3989 00000100:00000001:3.0:1787829189.479053:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4298663680 00000100:00080000:0.0:1787829189.479054:0:468:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787829189.479058:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829189.479059:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787829189.479061:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:3.0:1787829189.479061:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 00000020:00000001:0.0:1787829189.479064:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787829189.479067:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000020:00000001:0.0:1787829189.479069:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:3.0:1787829189.479069:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787829189.479073:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829189.479076:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:3.0:1787829189.479076:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787829189.479079:0:468:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787829189.479083:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829189.479083:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 02000000:00000001:0.0:1787829189.479087:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787829189.479089:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787829189.479092:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829189.479094:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00100000:3.0:1787829189.479095:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829189/real 1787829189] req@ffff8d30d4d270c0 x1874670646804864/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829214 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:0.0:1787829189.479096:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1787829189.479101:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829189.479104:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829189.479109:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787829189.479111:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829189.479116:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787829189.479118:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000400:00000001:3.0:1787829189.479119:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 02000000:00000001:0.0:1787829189.479120:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:3.0:1787829189.479124:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787829189.479126:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829189.479129:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00100000:0.0:1787829189.479133:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d24000 x1874670646805248/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787829189.479135:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829189.479138:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829189.479144:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787829189.479145:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787829189.479148:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829189.479148:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829189.479154:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829189.479158:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d270c0 x1874670646804864/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829214 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829189.479168:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829189.479172:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829189.479175:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829189.479177:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829189.479179:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829189.479181:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829189.479184:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829189.479187:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829189.479192:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829189.479196:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829189.479201:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d270c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646804864:192.168.203.143@tcp:250:kworker.0 00000100:00000001:3.0:1787829189.479206:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829189.479209:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829189.479210:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829189.479213:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829189.479216:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829189.479218:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829189.479220:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829189.479224:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829189.479226:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829189.479230:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829189.479231:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829189.479234:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829189.479240:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787829189.479242:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829189.479244:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1787829189.479246:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787829189.479248:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787829189.479249:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787829189.479251:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829189.479253:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787829189.479255:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829189.479257:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829189.479261:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d25a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646804992:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829189.479264:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1787829189.479266:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1787829189.479268:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829189.479270:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1787829189.479273:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829189.479279:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787829189.479281:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601343040 : -126234108208576 : ffff8d30d4d25a40) 00000100:00000001:3.0:1787829189.479285:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1787829189.479299:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:3.0:1787829189.479304:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787829189.479306:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829189.479308:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829189.479309:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1787829189.479312:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829189.479316:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298662422 00000100:00000001:3.0:1787829189.479319:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298663680 00000100:00000001:3.0:1787829189.479321:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298662422 00000100:00000001:3.0:1787829189.479324:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298663680 00000100:00000001:3.0:1787829189.479326:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829189.479328:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829189.479330:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1787829189.479334:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829189/real 1787829189] req@ffff8d30d4d25a40 x1874670646804992/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829214 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1787829189.479342:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787829189.479343:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829189.479346:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829189.479347:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829189.479349:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829189.479350:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829189.479352:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829189.479356:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d25a40 x1874670646804992/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829214 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829189.479380:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829189.479383:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829189.479386:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829189.479388:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829189.479389:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829189.479391:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829189.479393:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829189.479395:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829189.479399:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829189.479401:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829189.479404:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d25a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646804992:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829189.479408:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787829189.479410:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787829189.479411:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829189.479413:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787829189.479415:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829189.479417:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829189.479420:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d24f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646805120:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829189.479424:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1787829189.479426:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1787829189.479428:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829189.479430:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1787829189.479433:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829189.479438:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787829189.479439:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601340160 : -126234108211456 : ffff8d30d4d24f00) 00000100:00000001:3.0:1787829189.479443:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1787829189.479455:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:3.0:1787829189.479459:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787829189.479461:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829189.479462:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829189.479463:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1787829189.479465:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829189.479468:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298662422 00000100:00000001:3.0:1787829189.479469:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298663680 00000100:00000001:3.0:1787829189.479471:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298662422 00000100:00000001:3.0:1787829189.479472:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298663680 00000100:00000001:3.0:1787829189.479474:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829189.479475:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829189.479476:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1787829189.479479:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829189/real 1787829189] req@ffff8d30d4d24f00 x1874670646805120/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829214 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1787829189.479484:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787829189.479485:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829189.479486:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829189.479487:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829189.479489:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829189.479490:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829189.479491:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829189.479498:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d24f00 x1874670646805120/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829214 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829189.479503:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829189.479505:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829189.479507:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829189.479507:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829189.479508:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829189.479509:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829189.479510:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829189.479512:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829189.479514:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829189.479515:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829189.479518:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d24f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646805120:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829189.479521:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787829189.479522:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787829189.479523:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829189.479524:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787829189.479525:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829189.479526:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829189.479528:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d24000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646805248:192.168.203.143@tcp:8:kworker.0 00000100:00000001:3.0:1787829189.479530:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1787829189.479532:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1787829189.479533:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829189.479534:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1787829189.479536:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829189.479539:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787829189.479540:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601336320 : -126234108215296 : ffff8d30d4d24000) 00000100:00000001:3.0:1787829189.479543:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1787829189.479550:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:3.0:1787829189.479553:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787829189.479554:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829189.479555:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829189.479556:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1787829189.479558:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829189.479560:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298662435 00000100:00000001:3.0:1787829189.479561:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298663680 00000100:00000001:3.0:1787829189.479563:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298662435 00000100:00000001:3.0:1787829189.479564:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298663680 00000100:00000001:3.0:1787829189.479566:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829189.479568:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829189.479569:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1787829189.479573:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829189/real 1787829189] req@ffff8d30d4d24000 x1874670646805248/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829214 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1787829189.479588:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787829189.479589:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829189.479592:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829189.479593:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829189.479595:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829189.479597:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829189.479623:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829189.479628:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d24000 x1874670646805248/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829214 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829189.479633:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829189.479635:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829189.479637:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829189.479638:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829189.479639:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829189.479640:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829189.479641:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829189.479643:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829189.479645:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829189.479646:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829189.479648:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d24000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646805248:192.168.203.143@tcp:8:kworker.0 00000100:00000001:3.0:1787829189.479651:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829189.479652:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829189.479653:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829189.479655:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829189.479657:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829189.479658:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829189.479659:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829189.479660:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829189.479661:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829189.479663:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829189.479664:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829189.479665:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829189.479666:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829189.479667:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829189.479668:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829189.479669:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829189.479670:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829189.479671:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829189.479672:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829189.479673:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829189.479674:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829189.479675:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829189.479675:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829189.479676:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829189.479677:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829189.479678:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829189.479678:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829189.479679:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829189.479680:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829189.479681:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829189.479681:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829189.479683:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829189.479685:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787829189.479686:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829189.479687:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1787829189.479688:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829190.374700:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829190.374705:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829190.374762:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829190.374765:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829190.374768:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829191.398698:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829191.398706:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829191.398873:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829191.398883:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829191.398891:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829192.422728:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829192.422735:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829192.422835:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829192.422842:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829192.422846:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829193.446845:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829193.446871:0:815:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1787829193.446875:0:815:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1787829193.446881:0:815:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829193.446890:0:815:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1787829193.446893:0:815:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829193.446897:0:815:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829193.446901:0:815:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829193.446985:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829193.446992:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829193.446996:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829194.470776:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829194.470783:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829194.470892:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829194.470914:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829194.470918:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787829194.598734:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829194.598757:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1787829194.598761:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787829194.598766:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787829194.598772:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787829194.598798:0:40798:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 3995 00000100:00080000:2.0:1787829194.598812:0:40798:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787829194.598816:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829194.598817:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787829194.598822:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787829194.598824:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787829194.598827:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:2.0:1787829194.598829:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:2.0:1787829194.598831:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829194.598834:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:2.0:1787829194.598838:0:40798:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787829194.598840:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829194.598843:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787829194.598863:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787829194.598866:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829194.598868:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:2.0:1787829194.598871:0:40798:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829194.598886:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787829194.598887:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829194.598891:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787829194.598894:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787829194.598899:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829194.598912:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787829194.598922:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2ed00 x1874670646805376/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787829194.598932:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787829194.598936:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829194.598956:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787829194.598959:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829194.598962:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1787829194.598963:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787829194.598965:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787829194.598967:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787829194.598969:0:40798:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3995 00000100:00080000:2.0:1787829194.598982:0:40798:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787829194.598985:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829194.598986:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:3.0:1787829194.598994:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:2.0:1787829194.599008:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787829194.599012:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:3.0:1787829194.599014:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000020:00000001:2.0:1787829194.599016:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000020:00000001:2.0:1787829194.599019:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:3.0:1787829194.599019:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787829194.599024:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787829194.599026:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829194.599028:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829194.599029:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 02000000:00000001:3.0:1787829194.599037:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000100:00080000:2.0:1787829194.599038:0:40798:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787829194.599045:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829194.599045:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787829194.599059:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787829194.599060:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829194.599075:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646805376:192.168.203.143@tcp:250:kworker.0 02000000:00000001:2.0:1787829194.599076:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787829194.599083:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829194.599089:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:3.0:1787829194.599090:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1787829194.599095:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1787829194.599097:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829194.599099:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000008:00100000:2.0:1787829194.599104:0:40798:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 02000000:00000001:3.0:1787829194.599106:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:2.0:1787829194.599117:0:40798:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829194.599121:0:40798:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829194.599127:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787829194.599128:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829194.599130:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:3.0:1787829194.599131:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 02000000:00000001:2.0:1787829194.599132:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:3.0:1787829194.599134:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417879808 : -126234291671808 : ffff8d30c9e2ed00) 02000000:00000001:2.0:1787829194.599136:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829194.599147:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:2.0:1787829194.599150:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787829194.599158:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2f0c0 x1874670646805504/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787829194.599166:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787829194.599168:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829194.599172:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787829194.599174:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829194.599178:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1787829194.599180:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787829194.599185:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:3.0:1787829194.599186:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:2.0:1787829194.599195:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787829194.599200:0:40798:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3995 00000100:00080000:2.0:1787829194.599206:0:40798:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787829194.599206:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1787829194.599211:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829194.599213:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:3.0:1787829194.599214:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000020:00000001:2.0:1787829194.599222:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:3.0:1787829194.599223:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1787829194.599229:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:3.0:1787829194.599230:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000020:00000001:2.0:1787829194.599241:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000020:00000001:2.0:1787829194.599244:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:2.0:1787829194.599245:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829194.599246:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:2.0:1787829194.599249:0:40798:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787829194.599251:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829194.599254:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:3.0:1787829194.599255:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829194.599259:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829194.599260:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4298668800 00000100:00000001:3.0:1787829194.599285:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 02000000:00000001:2.0:1787829194.599286:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829194.599293:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4298668800 00000100:00000001:2.0:1787829194.599299:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:3.0:1787829194.599299:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 00000008:00100000:2.0:1787829194.599302:0:40798:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00000001:3.0:1787829194.599304:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000008:00000001:2.0:1787829194.599316:0:40798:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829194.599316:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829194.599331:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00000001:2.0:1787829194.599347:0:40798:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829194.599352:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829194/real 1787829194] req@ffff8d30c9e2ed00 x1874670646805376/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829219 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1787829194.599356:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787829194.599362:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829194.599371:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787829194.599376:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000400:00000001:3.0:1787829194.599378:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 02000000:00000001:2.0:1787829194.599380:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:3.0:1787829194.599386:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1787829194.599389:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829194.599395:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829194.599412:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00100000:2.0:1787829194.599415:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2c780 x1874670646805632/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787829194.599416:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829194.599426:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829194.599432:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829194.599434:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787829194.599440:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829194.599446:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829194.599447:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2ed00 x1874670646805376/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829219 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829194.599453:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829194.599460:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1787829194.599461:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787829194.599462:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787829194.599464:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787829194.599474:0:40798:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 3995 00000100:00000001:3.0:1787829194.599489:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829194.599493:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829194.599500:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829194.599502:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00080000:2.0:1787829194.599504:0:40798:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787829194.599504:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829194.599509:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787829194.599512:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829194.599513:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787829194.599520:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:3.0:1787829194.599521:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829194.599529:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000020:00000001:2.0:1787829194.599530:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787829194.599535:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:3.0:1787829194.599537:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000020:00000001:2.0:1787829194.599540:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00100000:3.0:1787829194.599544:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646805376:192.168.203.143@tcp:250:kworker.0 00000100:00000001:3.0:1787829194.599561:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:2.0:1787829194.599567:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:3.0:1787829194.599568:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829194.599571:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829194.599573:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 00000100:00000001:2.0:1787829194.599579:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 02000000:00000001:3.0:1787829194.599581:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00000001:2.0:1787829194.599583:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:3.0:1787829194.599585:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829194.599591:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 00000100:00080000:2.0:1787829194.599595:0:40798:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787829194.599600:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829194.599604:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787829194.599609:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787829194.599612:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829194.599615:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1787829194.599617:0:40798:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000008:00000001:2.0:1787829194.599620:0:40798:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829194.599623:0:40798:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829194.599629:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787829194.599631:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829194.599633:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787829194.599635:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787829194.599638:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829194.599640:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829194.599643:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1787829194.599647:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829194.599649:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829194.599655:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787829194.599659:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2de00 x1874670646805760/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787829194.599660:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829194.599667:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787829194.599668:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829194.599672:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1787829194.599674:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787829194.599675:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787829194.599676:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787829194.599678:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829194.599680:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787829194.599683:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829194.599685:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829194.599688:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646805504:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829194.599691:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1787829194.599693:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1787829194.599696:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829194.599699:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1787829194.599702:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829194.599709:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787829194.599710:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417880768 : -126234291670848 : ffff8d30c9e2f0c0) 00000100:00000001:3.0:1787829194.599715:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1787829194.599735:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:3.0:1787829194.599759:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787829194.599770:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829194.599771:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829194.599773:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1787829194.599776:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829194.599780:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298667543 00000100:00000001:3.0:1787829194.599788:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298668800 00000100:00000001:2.0:1787829194.599791:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787829194.599792:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298667543 00000100:00000001:2.0:1787829194.599801:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829194.599801:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298668800 00000100:00000001:3.0:1787829194.599811:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829194.599814:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829194.599816:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00000001:2.0:1787829194.599818:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829194.599821:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829194/real 1787829194] req@ffff8d30c9e2f0c0 x1874670646805504/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829219 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1787829194.599848:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787829194.599849:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829194.599852:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829194.599855:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829194.599857:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829194.599858:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829194.599859:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829194.599864:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2f0c0 x1874670646805504/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829219 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829194.599870:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829194.599871:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829194.599873:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829194.599875:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829194.599876:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829194.599879:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829194.599882:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829194.599885:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829194.599889:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829194.599891:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829194.599895:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646805504:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829194.599899:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787829194.599901:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787829194.599902:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829194.599905:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787829194.599907:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829194.599909:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829194.599913:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646805632:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829194.599917:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1787829194.599919:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1787829194.599920:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829194.599922:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1787829194.599926:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829194.599932:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787829194.599933:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417870208 : -126234291681408 : ffff8d30c9e2c780) 00000100:00000001:3.0:1787829194.599936:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1787829194.599945:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:3.0:1787829194.599949:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787829194.599951:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829194.599952:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829194.599953:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1787829194.599955:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829194.599958:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298667543 00000100:00000001:3.0:1787829194.599960:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298668800 00000100:00000001:3.0:1787829194.599961:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298667543 00000100:00000001:3.0:1787829194.599963:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298668800 00000100:00000001:3.0:1787829194.599964:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829194.599965:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829194.599967:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1787829194.599969:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829194/real 1787829194] req@ffff8d30c9e2c780 x1874670646805632/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829219 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1787829194.599974:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787829194.599975:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829194.599977:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829194.599978:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829194.599979:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829194.599991:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829194.599992:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829194.599994:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2c780 x1874670646805632/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829219 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829194.599999:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829194.600000:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829194.600002:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829194.600003:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829194.600004:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829194.600005:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829194.600006:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829194.600007:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829194.600009:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829194.600010:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829194.600013:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646805632:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829194.600016:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829194.600017:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829194.600018:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829194.600019:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829194.600020:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829194.600021:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829194.600022:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829194.600024:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829194.600024:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829194.600026:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829194.600027:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829194.600028:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829194.600029:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829194.600029:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829194.600030:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829194.600031:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829194.600032:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829194.600033:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829194.600034:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829194.600035:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829194.600036:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829194.600037:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829194.600039:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787829194.600040:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829194.600041:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1787829194.600042:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787829194.600043:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787829194.600044:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787829194.600045:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829194.600047:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787829194.600048:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829194.600049:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829194.600051:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646805760:192.168.203.143@tcp:8:kworker.0 00000100:00000001:3.0:1787829194.600053:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1787829194.600055:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1787829194.600056:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829194.600057:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1787829194.600059:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829194.600062:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787829194.600063:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417875968 : -126234291675648 : ffff8d30c9e2de00) 00000100:00000001:3.0:1787829194.600065:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1787829194.600072:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:3.0:1787829194.600075:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787829194.600076:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829194.600077:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829194.600078:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1787829194.600080:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829194.600082:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298667556 00000100:00000001:3.0:1787829194.600084:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298668800 00000100:00000001:3.0:1787829194.600085:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298667556 00000100:00000001:3.0:1787829194.600087:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298668800 00000100:00000001:3.0:1787829194.600088:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829194.600089:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829194.600090:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1787829194.600093:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829194/real 1787829194] req@ffff8d30c9e2de00 x1874670646805760/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829219 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1787829194.600097:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787829194.600098:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829194.600099:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829194.600100:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829194.600101:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829194.600102:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829194.600103:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829194.600105:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2de00 x1874670646805760/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829219 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829194.600109:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829194.600111:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829194.600113:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829194.600114:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829194.600115:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829194.600116:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829194.600117:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829194.600119:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829194.600121:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829194.600122:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829194.600124:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646805760:192.168.203.143@tcp:8:kworker.0 00000100:00000001:3.0:1787829194.600126:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829194.600127:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829194.600128:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829194.600129:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829194.600130:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829194.600131:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829194.600131:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829194.600133:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829194.600133:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829194.600136:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829194.600137:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829194.600138:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829194.600139:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787829194.600140:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829194.600141:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1787829194.600143:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829195.494764:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829195.494788:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829195.494850:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829195.494856:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829195.494859:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829196.518840:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829196.518850:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829196.518952:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829196.518960:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829196.518968:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829197.542838:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829197.542858:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829197.543064:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829197.543075:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829197.543083:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829198.566779:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829198.566786:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829198.566857:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829198.566865:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829198.566871:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829199.590753:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829199.590759:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829199.590830:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829199.590833:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829199.590836:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787829199.718702:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829199.718722:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1787829199.718724:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787829199.718727:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787829199.718731:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787829199.718736:0:40798:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 4000 00000100:00080000:2.0:1787829199.718746:0:40798:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787829199.718748:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829199.718750:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787829199.718753:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787829199.718755:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787829199.718758:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:2.0:1787829199.718759:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:2.0:1787829199.718761:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829199.718762:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:2.0:1787829199.718765:0:40798:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787829199.718767:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829199.718769:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787829199.718772:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787829199.718774:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829199.718776:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:2.0:1787829199.718778:0:40798:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829199.718788:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787829199.718789:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829199.718792:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787829199.718794:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787829199.718798:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829199.718807:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787829199.718814:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2c3c0 x1874670646805888/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787829199.718822:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787829199.718826:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829199.718838:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787829199.718840:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829199.718843:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1787829199.718844:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787829199.718845:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787829199.718847:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787829199.718848:0:40798:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4000 00000100:00080000:2.0:1787829199.718850:0:40798:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787829199.718851:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829199.718852:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787829199.718854:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787829199.718854:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787829199.718856:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000020:00000001:2.0:1787829199.718857:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:2.0:1787829199.718859:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829199.718860:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:2.0:1787829199.718862:0:40798:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787829199.718864:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829199.718865:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787829199.718869:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787829199.718870:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829199.718871:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1787829199.718875:0:40798:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1787829199.718878:0:40798:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829199.718880:0:40798:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829199.718882:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787829199.718883:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829199.718885:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787829199.718885:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787829199.718887:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829199.718887:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787829199.718893:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829199.718895:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00100000:2.0:1787829199.718898:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2fc00 x1874670646806016/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787829199.718900:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787829199.718904:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787829199.718908:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787829199.718908:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829199.718911:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829199.718915:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829199.718916:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000100:00080000:2.0:1787829199.718917:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 02000000:00000001:3.0:1787829199.718920:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00080000:2.0:1787829199.718921:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1787829199.718924:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787829199.718927:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 02000000:00000001:3.0:1787829199.718927:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829199.718929:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787829199.718936:0:40798:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4000 00000100:00100000:3.0:1787829199.718938:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646805888:192.168.203.143@tcp:250:kworker.0 00000100:00080000:2.0:1787829199.718939:0:40798:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787829199.718942:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829199.718944:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787829199.718947:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787829199.718949:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:3.0:1787829199.718949:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000020:00000001:2.0:1787829199.718953:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000020:00000001:2.0:1787829199.718954:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 02000000:00000001:3.0:1787829199.718955:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:2.0:1787829199.718956:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829199.718959:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 02000000:00000001:3.0:1787829199.718960:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787829199.718963:0:40798:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787829199.718973:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829199.718973:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1787829199.718977:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787829199.718979:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787829199.718981:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829199.718981:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829199.718986:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1787829199.718988:0:40798:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1787829199.718989:0:40798:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829199.718990:0:40798:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829199.718994:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787829199.718994:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829199.718995:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787829199.718996:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:3.0:1787829199.719000:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 02000000:00000001:2.0:1787829199.719001:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829199.719005:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417869248 : -126234291682368 : ffff8d30c9e2c3c0) 00000100:00000001:2.0:1787829199.719008:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829199.719011:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00100000:2.0:1787829199.719017:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2d2c0 x1874670646806144/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787829199.719023:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787829199.719024:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829199.719026:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787829199.719028:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829199.719031:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1787829199.719032:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787829199.719033:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787829199.719035:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787829199.719037:0:40798:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4000 00000100:00080000:2.0:1787829199.719039:0:40798:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787829199.719041:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829199.719041:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787829199.719043:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787829199.719044:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787829199.719045:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000020:00000001:2.0:1787829199.719048:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000800:00000100:3.0:1787829199.719051:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:2.0:1787829199.719052:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829199.719054:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:2.0:1787829199.719058:0:40798:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787829199.719061:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829199.719062:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787829199.719064:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787829199.719066:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829199.719068:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:3.0:1787829199.719068:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000008:00100000:2.0:1787829199.719072:0:40798:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000008:00000001:2.0:1787829199.719074:0:40798:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829199.719076:0:40798:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829199.719076:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829199.719081:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829199.719082:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787829199.719086:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829199.719087:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1787829199.719088:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787829199.719091:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787829199.719092:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829199.719095:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829199.719096:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787829199.719103:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2cb40 x1874670646806272/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787829199.719103:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4298673920 00000100:00000001:2.0:1787829199.719112:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787829199.719113:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829199.719115:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829199.719117:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 00000100:00000001:3.0:1787829199.719133:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4298673920 00000100:00000001:3.0:1787829199.719138:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 00000100:00000001:3.0:1787829199.719144:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829199.719146:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829199.719148:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1787829199.719155:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829199/real 1787829199] req@ffff8d30c9e2c3c0 x1874670646805888/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829224 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1787829199.719172:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787829199.719174:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829199.719178:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829199.719180:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829199.719183:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829199.719186:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829199.719188:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829199.719205:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2c3c0 x1874670646805888/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829224 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829199.719213:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829199.719216:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829199.719219:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829199.719220:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829199.719222:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829199.719224:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829199.719226:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829199.719235:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829199.719239:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829199.719242:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829199.719246:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646805888:192.168.203.143@tcp:250:kworker.0 00000100:00000001:3.0:1787829199.719251:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829199.719253:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829199.719255:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829199.719257:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829199.719261:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829199.719263:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829199.719264:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829199.719268:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829199.719270:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829199.719273:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829199.719274:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829199.719277:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829199.719283:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787829199.719285:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829199.719287:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1787829199.719289:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787829199.719290:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787829199.719303:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787829199.719304:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829199.719306:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787829199.719308:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829199.719310:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829199.719313:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646806016:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829199.719317:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1787829199.719319:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1787829199.719321:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829199.719323:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1787829199.719326:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829199.719331:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787829199.719333:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417883648 : -126234291667968 : ffff8d30c9e2fc00) 00000100:00000001:3.0:1787829199.719337:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1787829199.719350:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:3.0:1787829199.719355:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787829199.719369:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829199.719371:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829199.719372:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1787829199.719375:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829199.719379:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298672662 00000100:00000001:3.0:1787829199.719382:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298673920 00000100:00000001:3.0:1787829199.719385:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298672662 00000100:00000001:3.0:1787829199.719387:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298673920 00000100:00000001:3.0:1787829199.719389:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829199.719392:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829199.719394:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1787829199.719398:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829199/real 1787829199] req@ffff8d30c9e2fc00 x1874670646806016/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829224 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1787829199.719408:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787829199.719409:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829199.719412:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829199.719413:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829199.719415:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829199.719417:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829199.719419:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829199.719423:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2fc00 x1874670646806016/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829224 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829199.719430:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829199.719432:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829199.719434:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829199.719436:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829199.719437:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829199.719439:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829199.719441:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829199.719443:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829199.719446:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829199.719449:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829199.719452:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646806016:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829199.719456:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787829199.719457:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787829199.719459:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829199.719461:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787829199.719463:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829199.719465:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829199.719468:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2d2c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646806144:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829199.719472:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1787829199.719473:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1787829199.719475:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829199.719477:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1787829199.719480:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829199.719485:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787829199.719487:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417873088 : -126234291678528 : ffff8d30c9e2d2c0) 00000100:00000001:3.0:1787829199.719490:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1787829199.719501:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:3.0:1787829199.719505:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787829199.719508:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829199.719509:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829199.719511:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1787829199.719514:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829199.719517:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298672662 00000100:00000001:3.0:1787829199.719520:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298673920 00000100:00000001:3.0:1787829199.719522:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298672662 00000100:00000001:3.0:1787829199.719524:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298673920 00000100:00000001:3.0:1787829199.719526:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829199.719528:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829199.719530:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1787829199.719534:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829199/real 1787829199] req@ffff8d30c9e2d2c0 x1874670646806144/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829224 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1787829199.719542:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787829199.719543:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829199.719545:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829199.719547:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829199.719548:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829199.719550:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829199.719551:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829199.719555:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2d2c0 x1874670646806144/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829224 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829199.719562:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829199.719564:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829199.719567:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829199.719568:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829199.719569:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829199.719571:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829199.719573:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829199.719575:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829199.719578:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829199.719580:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829199.719583:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2d2c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646806144:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829199.719587:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787829199.719588:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787829199.719590:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829199.719592:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787829199.719594:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829199.719595:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829199.719625:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646806272:192.168.203.143@tcp:8:kworker.0 00000100:00000001:3.0:1787829199.719628:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1787829199.719629:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1787829199.719630:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829199.719631:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1787829199.719634:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829199.719665:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787829199.719666:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417871168 : -126234291680448 : ffff8d30c9e2cb40) 00000100:00000001:3.0:1787829199.719669:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1787829199.719680:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:3.0:1787829199.719686:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787829199.719689:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829199.719690:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829199.719692:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1787829199.719695:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829199.719699:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298672676 00000100:00000001:3.0:1787829199.719702:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298673920 00000100:00000001:3.0:1787829199.719704:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298672676 00000100:00000001:3.0:1787829199.719707:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298673920 00000100:00000001:3.0:1787829199.719739:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829199.719741:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829199.719743:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1787829199.719748:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829199/real 1787829199] req@ffff8d30c9e2cb40 x1874670646806272/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829224 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1787829199.719780:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787829199.719782:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829199.719785:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829199.719786:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829199.719788:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829199.719790:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829199.719791:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829199.719796:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2cb40 x1874670646806272/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829224 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829199.719803:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829199.719806:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829199.719808:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829199.719810:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829199.719811:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829199.719813:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829199.719815:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829199.719817:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829199.719821:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829199.719823:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829199.719826:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646806272:192.168.203.143@tcp:8:kworker.0 00000100:00000001:3.0:1787829199.719830:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829199.719832:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829199.719834:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829199.719835:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829199.719838:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829199.719840:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829199.719841:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829199.719843:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829199.719845:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829199.719848:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829199.719849:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829199.719851:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829199.719852:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829199.719854:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829199.719856:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829199.719857:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829199.719858:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829199.719860:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829199.719861:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829199.719863:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829199.719865:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829199.719866:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829199.719867:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829199.719868:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829199.719870:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829199.719871:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829199.719872:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829199.719873:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829199.719874:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829199.719876:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829199.719877:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829199.719879:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829199.719882:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787829199.719884:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829199.719886:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1787829199.719887:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829200.614681:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829200.614687:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829200.614737:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829200.614740:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829200.614743:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829201.638737:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829201.638742:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829201.638786:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829201.638790:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829201.638793:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829202.662674:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829202.662681:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829202.662755:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829202.662760:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829202.662764:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829203.686710:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829203.686719:0:815:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1787829203.686721:0:815:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1787829203.686724:0:815:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829203.686731:0:815:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1787829203.686734:0:815:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829203.686737:0:815:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829203.686740:0:815:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829203.686799:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829203.686803:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829203.686807:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829204.710735:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829204.710741:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829204.710852:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829204.710856:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829204.710860:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787829204.838739:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829204.838747:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1787829204.838749:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787829204.838753:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787829204.838756:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787829204.838761:0:40798:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 4005 00000100:00080000:2.0:1787829204.838770:0:40798:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787829204.838773:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829204.838774:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787829204.838777:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787829204.838779:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787829204.838782:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:2.0:1787829204.838783:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:2.0:1787829204.838785:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829204.838786:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:2.0:1787829204.838789:0:40798:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787829204.838791:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829204.838793:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787829204.838796:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787829204.838798:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829204.838799:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:2.0:1787829204.838801:0:40798:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829204.838810:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787829204.838811:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829204.838814:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787829204.838817:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787829204.838821:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829204.838831:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787829204.838839:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2d680 x1874670646806400/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787829204.838847:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787829204.838851:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829204.838870:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787829204.838874:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829204.838878:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1787829204.838879:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787829204.838881:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787829204.838884:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787829204.838887:0:40798:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4005 00000100:00080000:2.0:1787829204.838892:0:40798:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787829204.838894:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829204.838896:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787829204.838899:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787829204.838900:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787829204.838903:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000020:00000001:2.0:1787829204.838905:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:1.0:1787829204.838908:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787829204.838908:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829204.838911:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:1.0:1787829204.838914:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00080000:2.0:1787829204.838918:0:40798:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:1.0:1787829204.838919:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787829204.838921:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787829204.838925:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829204.838925:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829204.838929:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787829204.838930:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1787829204.838932:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829204.838935:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829204.838938:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00100000:1.0:1787829204.838940:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2d680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646806400:192.168.203.143@tcp:250:kworker.0 02000000:00000001:2.0:1787829204.838943:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829204.838947:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:2.0:1787829204.838948:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 02000000:00000001:1.0:1787829204.838952:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000008:00100000:2.0:1787829204.838954:0:40798:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 02000000:00000001:1.0:1787829204.838955:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829204.838960:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000008:00000001:2.0:1787829204.838962:0:40798:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829204.838966:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829204.838967:0:40798:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829204.838978:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787829204.838980:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829204.838982:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787829204.838982:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787829204.838984:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:1.0:1787829204.838985:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417874048 : -126234291677568 : ffff8d30c9e2d680) 02000000:00000001:2.0:1787829204.838989:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829204.838990:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:2.0:1787829204.838999:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787829204.839008:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2e1c0 x1874670646806528/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787829204.839022:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:2.0:1787829204.839028:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787829204.839031:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829204.839034:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1787829204.839036:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829204.839039:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829204.839042:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787829204.839042:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1787829204.839045:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00080000:2.0:1787829204.839050:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:1.0:1787829204.839051:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829204.839053:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:1.0:1787829204.839056:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4298679040 00000100:00080000:2.0:1787829204.839057:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787829204.839062:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:1.0:1787829204.839064:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 00000100:00080000:2.0:1787829204.839066:0:40798:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4005 00000100:00080000:2.0:1787829204.839070:0:40798:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787829204.839072:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829204.839073:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787829204.839077:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787829204.839078:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787829204.839080:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:1.0:1787829204.839081:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4298679040 00000020:00000001:2.0:1787829204.839082:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:1.0:1787829204.839086:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 00000100:00000001:2.0:1787829204.839086:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829204.839087:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:1.0:1787829204.839093:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00080000:2.0:1787829204.839094:0:40798:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:1.0:1787829204.839098:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829204.839099:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829204.839102:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 02000000:00000001:2.0:1787829204.839104:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00100000:1.0:1787829204.839109:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829204/real 1787829204] req@ffff8d30c9e2d680 x1874670646806400/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829229 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1787829204.839110:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000001:1.0:1787829204.839119:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787829204.839120:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787829204.839123:0:2390:0:(events.c:78:reply_in_callback()) Process entered 02000000:00000001:2.0:1787829204.839124:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829204.839126:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787829204.839129:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:1.0:1787829204.839130:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000008:00100000:2.0:1787829204.839134:0:40798:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00000001:1.0:1787829204.839135:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829204.839138:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000008:00000001:2.0:1787829204.839140:0:40798:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829204.839145:0:40798:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829204.839146:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2d680 x1874670646806400/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829229 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1787829204.839154:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787829204.839158:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829204.839159:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829204.839163:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:1.0:1787829204.839165:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 02000000:00000001:2.0:1787829204.839166:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787829204.839170:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829204.839171:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829204.839175:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787829204.839177:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829204.839178:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829204.839184:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787829204.839186:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787829204.839190:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:2.0:1787829204.839191:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2cb40 x1874670646806656/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829204.839196:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787829204.839200:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787829204.839206:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2d680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646806400:192.168.203.143@tcp:250:kworker.0 00000100:00000001:2.0:1787829204.839212:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787829204.839213:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829204.839216:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829204.839217:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829204.839221:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787829204.839223:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 00000100:00000001:2.0:1787829204.839223:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829204.839228:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00080000:2.0:1787829204.839229:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000020:00000001:1.0:1787829204.839231:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787829204.839234:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 00000100:00080000:2.0:1787829204.839237:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 02000000:00000001:1.0:1787829204.839239:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 00000100:00000001:2.0:1787829204.839241:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 02000000:00000001:1.0:1787829204.839242:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00080000:2.0:1787829204.839246:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787829204.839247:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787829204.839250:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829204.839252:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829204.839252:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787829204.839259:0:40798:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4005 00000100:00000001:1.0:1787829204.839260:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787829204.839264:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829204.839267:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00080000:2.0:1787829204.839267:0:40798:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1787829204.839270:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1787829204.839273:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787829204.839274:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787829204.839274:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787829204.839278:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829204.839279:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 02000000:00000001:1.0:1787829204.839280:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787829204.839283:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000020:00000001:2.0:1787829204.839285:0:40798:0:(genops.c:792:class_conn2export()) Process entered 02000000:00000001:1.0:1787829204.839286:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1787829204.839288:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00100000:1.0:1787829204.839291:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646806528:192.168.203.143@tcp:38:kworker.0 00000020:00000001:2.0:1787829204.839293:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:1.0:1787829204.839295:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787829204.839298:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000020:00000001:2.0:1787829204.839298:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 02000000:00000001:1.0:1787829204.839301:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829204.839304:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:2.0:1787829204.839304:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 02000000:00000001:1.0:1787829204.839309:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829204.839309:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:1.0:1787829204.839317:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00080000:2.0:1787829204.839318:0:40798:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:1.0:1787829204.839319:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417876928 : -126234291674688 : ffff8d30c9e2e1c0) 00000100:00000001:2.0:1787829204.839325:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829204.839326:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 02000000:00000001:2.0:1787829204.839331:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787829204.839335:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787829204.839338:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:1.0:1787829204.839340:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:2.0:1787829204.839343:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:1.0:1787829204.839345:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000008:00100000:2.0:1787829204.839348:0:40798:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00000001:1.0:1787829204.839349:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829204.839351:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829204.839354:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000008:00000001:2.0:1787829204.839357:0:40798:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829204.839358:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829204.839363:0:40798:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829204.839364:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298677782 00000100:00000001:1.0:1787829204.839368:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298679040 00000100:00000001:1.0:1787829204.839371:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298677782 02000000:00000001:2.0:1787829204.839371:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:1.0:1787829204.839375:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298679040 02000000:00000001:2.0:1787829204.839375:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829204.839381:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787829204.839381:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:1.0:1787829204.839386:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829204.839386:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:1.0:1787829204.839391:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 02000000:00000001:2.0:1787829204.839391:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829204.839396:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829204/real 1787829204] req@ffff8d30c9e2e1c0 x1874670646806528/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829229 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787829204.839400:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:1.0:1787829204.839406:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787829204.839407:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00100000:2.0:1787829204.839410:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2d2c0 x1874670646806784/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787829204.839411:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787829204.839413:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787829204.839416:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829204.839420:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829204.839422:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829204.839428:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00100000:1.0:1787829204.839429:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2e1c0 x1874670646806528/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829229 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787829204.839432:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829204.839438:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829204.839439:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829204.839442:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787829204.839444:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829204.839446:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829204.839447:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829204.839448:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787829204.839450:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787829204.839451:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829204.839453:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787829204.839455:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787829204.839458:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646806528:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787829204.839461:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787829204.839462:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787829204.839464:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829204.839465:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787829204.839466:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829204.839467:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829204.839469:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646806656:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787829204.839471:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787829204.839472:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787829204.839474:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829204.839475:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787829204.839478:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829204.839483:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787829204.839484:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417871168 : -126234291680448 : ffff8d30c9e2cb40) 00000100:00000001:1.0:1787829204.839487:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1787829204.839498:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787829204.839503:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787829204.839505:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829204.839507:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829204.839508:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787829204.839511:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829204.839515:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298677782 00000100:00000001:1.0:1787829204.839517:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298679040 00000100:00000001:1.0:1787829204.839520:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298677782 00000100:00000001:1.0:1787829204.839522:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298679040 00000100:00000001:1.0:1787829204.839524:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829204.839527:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829204.839528:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787829204.839532:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829204/real 1787829204] req@ffff8d30c9e2cb40 x1874670646806656/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829229 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787829204.839537:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787829204.839538:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787829204.839540:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787829204.839541:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787829204.839542:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829204.839544:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829204.839545:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787829204.839547:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2cb40 x1874670646806656/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829229 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787829204.839552:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829204.839553:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787829204.839555:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829204.839555:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829204.839556:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829204.839557:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787829204.839558:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787829204.839560:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829204.839562:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787829204.839563:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787829204.839565:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646806656:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787829204.839567:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829204.839569:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829204.839570:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787829204.839571:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787829204.839573:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787829204.839574:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787829204.839575:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787829204.839576:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787829204.839577:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787829204.839580:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787829204.839581:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829204.839583:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829204.839583:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787829204.839584:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787829204.839585:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787829204.839586:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787829204.839587:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787829204.839588:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787829204.839588:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787829204.839590:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787829204.839591:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829204.839592:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829204.839595:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787829204.839596:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829204.839597:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787829204.839598:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1787829204.839599:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787829204.839624:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787829204.839625:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829204.839627:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787829204.839628:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829204.839630:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829204.839632:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2d2c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646806784:192.168.203.143@tcp:8:kworker.0 00000100:00000001:1.0:1787829204.839634:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787829204.839636:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787829204.839637:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829204.839638:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787829204.839641:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829204.839645:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787829204.839646:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417873088 : -126234291678528 : ffff8d30c9e2d2c0) 00000100:00000001:1.0:1787829204.839648:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1787829204.839657:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787829204.839660:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787829204.839661:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829204.839662:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829204.839663:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787829204.839665:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829204.839667:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298677796 00000100:00000001:1.0:1787829204.839669:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298679040 00000100:00000001:1.0:1787829204.839670:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298677796 00000100:00000001:1.0:1787829204.839672:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298679040 00000100:00000001:1.0:1787829204.839691:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829204.839692:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829204.839693:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787829204.839697:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829204/real 1787829204] req@ffff8d30c9e2d2c0 x1874670646806784/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829229 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787829204.839703:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787829204.839704:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787829204.839706:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787829204.839707:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787829204.839707:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829204.839709:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829204.839710:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787829204.839712:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2d2c0 x1874670646806784/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829229 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787829204.839717:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829204.839718:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787829204.839720:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829204.839721:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829204.839722:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829204.839723:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787829204.839724:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787829204.839725:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829204.839727:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787829204.839728:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787829204.839730:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2d2c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646806784:192.168.203.143@tcp:8:kworker.0 00000100:00000001:1.0:1787829204.839733:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829204.839734:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829204.839735:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787829204.839736:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787829204.839737:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787829204.839738:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787829204.839739:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787829204.839741:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787829204.839742:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787829204.839743:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787829204.839744:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829204.839745:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829204.839747:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787829204.839748:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829204.839750:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787829204.839752:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829205.734660:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829205.734665:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829205.734731:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829205.734734:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829205.734737:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829206.758704:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829206.758709:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829206.758781:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829206.758784:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829206.758787:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829207.782705:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829207.782711:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829207.782761:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829207.782764:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829207.782767:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829208.806705:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829208.806712:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829208.806777:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829208.806781:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829208.806784:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829209.830743:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829209.830750:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829209.830822:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829209.830828:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829209.830831:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787829209.958694:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829209.958702:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1787829209.958704:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1787829209.958707:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787829209.958711:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1787829209.958716:0:468:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 4010 00000100:00080000:1.0:1787829209.958725:0:468:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1787829209.958727:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787829209.958729:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:1.0:1787829209.958732:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:1.0:1787829209.958734:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:1.0:1787829209.958743:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:1.0:1787829209.958745:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:1.0:1787829209.958746:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787829209.958747:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:1.0:1787829209.958751:0:468:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:1.0:1787829209.958753:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829209.958755:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1787829209.958758:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829209.958760:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829209.958761:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:1.0:1787829209.958763:0:468:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829209.958773:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1787829209.958774:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829209.958777:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1787829209.958779:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1787829209.958783:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829209.958791:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829209.958799:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d13ef480 x1874670646806912/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787829209.958806:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787829209.958809:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829209.958823:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1787829209.958826:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829209.958828:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1787829209.958829:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1787829209.958830:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787829209.958832:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1787829209.958834:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4010 00000100:00080000:1.0:1787829209.958836:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1787829209.958838:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787829209.958839:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:1.0:1787829209.958840:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:1.0:1787829209.958841:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:1.0:1787829209.958842:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000020:00000001:1.0:1787829209.958844:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:1.0:1787829209.958846:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787829209.958847:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:1.0:1787829209.958849:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:1.0:1787829209.958851:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829209.958852:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1787829209.958856:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829209.958858:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829209.958859:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:1.0:1787829209.958863:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1787829209.958866:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829209.958867:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829209.958871:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1787829209.958873:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829209.958875:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1787829209.958876:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1787829209.958877:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829209.958881:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829209.958884:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d13ee580 x1874670646807040/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787829209.958892:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787829209.958893:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787829209.958894:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829209.958902:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829209.958902:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787829209.958907:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00080000:1.0:1787829209.958908:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829209.958913:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1787829209.958913:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787829209.958918:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829209.958919:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1787829209.958928:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 02000000:00000001:3.0:1787829209.958930:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:1.0:1787829209.958932:0:468:0:(import.c:485:import_select_connection()) Process entered 02000000:00000001:3.0:1787829209.958934:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00080000:1.0:1787829209.958939:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4010 02000000:00000001:3.0:1787829209.958943:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1787829209.958944:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1787829209.958950:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787829209.958954:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00100000:3.0:1787829209.958955:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d13ef480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646806912:192.168.203.143@tcp:250:kworker.0 00000020:00000001:1.0:1787829209.958957:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:1.0:1787829209.958962:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:1.0:1787829209.958967:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:3.0:1787829209.958967:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000020:00000001:1.0:1787829209.958976:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 02000000:00000001:3.0:1787829209.958978:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:1.0:1787829209.958979:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 02000000:00000001:3.0:1787829209.958988:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829209.958991:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:1.0:1787829209.958994:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 02000000:00000001:3.0:1787829209.958999:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1787829209.959002:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:1.0:1787829209.959006:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829209.959008:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1787829209.959012:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829209.959014:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829209.959015:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:1.0:1787829209.959017:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1787829209.959019:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829209.959019:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787829209.959024:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839541351552 : -126234168200064 : ffff8d30d13ef480) 00000100:00000001:1.0:1787829209.959025:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829209.959036:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:3.0:1787829209.959038:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 02000000:00000001:1.0:1787829209.959039:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829209.959043:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1787829209.959044:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1787829209.959046:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829209.959050:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829209.959054:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d13ee1c0 x1874670646807168/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787829209.959060:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787829209.959061:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829209.959062:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1787829209.959064:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829209.959067:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1787829209.959069:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1787829209.959070:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787829209.959072:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1787829209.959074:0:468:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4010 00000100:00080000:1.0:1787829209.959076:0:468:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1787829209.959082:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787829209.959087:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000800:00000100:3.0:1787829209.959088:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000020:00000001:1.0:1787829209.959097:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:1.0:1787829209.959099:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:1.0:1787829209.959101:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000020:00000001:1.0:1787829209.959105:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:1.0:1787829209.959107:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787829209.959109:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:3.0:1787829209.959111:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00080000:1.0:1787829209.959113:0:468:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:1.0:1787829209.959116:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829209.959118:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:3.0:1787829209.959118:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 02000000:00000001:1.0:1787829209.959122:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829209.959123:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829209.959134:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829209.959135:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787829209.959143:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:3.0:1787829209.959148:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000008:00100000:1.0:1787829209.959149:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000008:00000001:1.0:1787829209.959159:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829209.959161:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829209.959162:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4298684160 02000000:00000001:1.0:1787829209.959166:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1787829209.959168:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829209.959170:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1787829209.959171:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1787829209.959174:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829209.959174:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 00000100:00000001:1.0:1787829209.959181:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829209.959186:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d13eda40 x1874670646807296/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787829209.959192:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787829209.959193:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829209.959195:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829209.959197:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4298684160 00000100:00000001:3.0:1787829209.959205:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 00000100:00000001:3.0:1787829209.959211:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829209.959213:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829209.959217:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1787829209.959224:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829209/real 1787829209] req@ffff8d30d13ef480 x1874670646806912/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829234 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1787829209.959233:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787829209.959235:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829209.959242:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829209.959246:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829209.959247:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829209.959249:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829209.959251:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829209.959255:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d13ef480 x1874670646806912/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829234 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829209.959261:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829209.959265:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829209.959267:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829209.959268:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829209.959269:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829209.959271:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829209.959272:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829209.959275:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829209.959278:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829209.959280:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829209.959282:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d13ef480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646806912:192.168.203.143@tcp:250:kworker.0 00000100:00000001:3.0:1787829209.959291:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829209.959293:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829209.959294:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829209.959297:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829209.959300:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829209.959302:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829209.959304:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829209.959309:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829209.959311:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829209.959314:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829209.959315:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829209.959317:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829209.959325:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787829209.959326:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829209.959328:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1787829209.959330:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787829209.959331:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787829209.959332:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787829209.959333:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829209.959335:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787829209.959336:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829209.959338:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829209.959340:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d13ee580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646807040:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829209.959342:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1787829209.959343:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1787829209.959345:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829209.959346:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1787829209.959348:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829209.959353:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787829209.959354:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839541347712 : -126234168203904 : ffff8d30d13ee580) 00000100:00000001:3.0:1787829209.959358:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1787829209.959368:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:3.0:1787829209.959371:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787829209.959372:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829209.959373:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829209.959374:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1787829209.959376:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829209.959380:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298682902 00000100:00000001:3.0:1787829209.959383:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298684160 00000100:00000001:3.0:1787829209.959384:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298682902 00000100:00000001:3.0:1787829209.959386:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298684160 00000100:00000001:3.0:1787829209.959388:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829209.959389:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829209.959390:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1787829209.959393:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829209/real 1787829209] req@ffff8d30d13ee580 x1874670646807040/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829234 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1787829209.959399:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787829209.959400:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829209.959402:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829209.959403:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829209.959404:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829209.959405:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829209.959406:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829209.959408:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d13ee580 x1874670646807040/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829234 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829209.959415:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829209.959417:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829209.959420:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829209.959421:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829209.959423:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829209.959425:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829209.959434:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829209.959437:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829209.959441:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829209.959444:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829209.959448:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d13ee580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646807040:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829209.959452:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787829209.959454:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787829209.959455:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829209.959457:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787829209.959459:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829209.959465:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829209.959469:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d13ee1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646807168:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829209.959473:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1787829209.959475:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1787829209.959477:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829209.959479:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1787829209.959482:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829209.959488:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787829209.959489:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839541346752 : -126234168204864 : ffff8d30d13ee1c0) 00000100:00000001:3.0:1787829209.959494:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1787829209.959507:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:3.0:1787829209.959512:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787829209.959515:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829209.959516:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829209.959518:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1787829209.959520:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829209.959523:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298682902 00000100:00000001:3.0:1787829209.959525:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298684160 00000100:00000001:3.0:1787829209.959527:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298682902 00000100:00000001:3.0:1787829209.959529:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298684160 00000100:00000001:3.0:1787829209.959530:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829209.959532:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829209.959533:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1787829209.959536:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829209/real 1787829209] req@ffff8d30d13ee1c0 x1874670646807168/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829234 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1787829209.959541:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787829209.959542:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829209.959544:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829209.959545:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829209.959546:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829209.959547:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829209.959548:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829209.959551:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d13ee1c0 x1874670646807168/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829234 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829209.959555:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829209.959557:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829209.959559:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829209.959560:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829209.959561:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829209.959562:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829209.959563:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829209.959565:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829209.959567:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829209.959568:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829209.959570:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d13ee1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646807168:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829209.959572:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787829209.959573:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787829209.959574:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829209.959576:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787829209.959577:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829209.959578:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829209.959580:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d13eda40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646807296:192.168.203.143@tcp:8:kworker.0 00000100:00000001:3.0:1787829209.959582:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1787829209.959583:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1787829209.959584:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829209.959586:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1787829209.959588:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829209.959592:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787829209.959593:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839541344832 : -126234168206784 : ffff8d30d13eda40) 00000100:00000001:3.0:1787829209.959595:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1787829209.959654:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:3.0:1787829209.959658:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787829209.959660:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829209.959662:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829209.959663:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1787829209.959665:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829209.959668:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298682916 00000100:00000001:3.0:1787829209.959670:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298684160 00000100:00000001:3.0:1787829209.959672:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298682916 00000100:00000001:3.0:1787829209.959674:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298684160 00000100:00000001:3.0:1787829209.959676:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829209.959677:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829209.959679:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1787829209.959682:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829209/real 1787829209] req@ffff8d30d13eda40 x1874670646807296/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829234 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1787829209.959691:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787829209.959692:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829209.959695:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829209.959696:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829209.959698:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829209.959700:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829209.959701:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829209.959705:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d13eda40 x1874670646807296/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829234 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829209.959713:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829209.959716:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829209.959719:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829209.959721:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829209.959722:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829209.959724:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829209.959726:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829209.959729:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829209.959734:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829209.959736:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829209.959738:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d13eda40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646807296:192.168.203.143@tcp:8:kworker.0 00000100:00000001:3.0:1787829209.959741:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829209.959742:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829209.959744:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829209.959745:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829209.959747:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829209.959748:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829209.959749:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829209.959752:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829209.959753:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829209.959756:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829209.959757:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829209.959758:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829209.959759:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829209.959760:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829209.959761:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829209.959762:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829209.959762:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829209.959763:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829209.959764:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829209.959765:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829209.959766:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829209.959766:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829209.959767:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829209.959768:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829209.959769:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829209.959770:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829209.959770:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829209.959771:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829209.959772:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829209.959773:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829209.959774:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829209.959775:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829209.959778:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787829209.959779:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829209.959780:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1787829209.959781:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829210.854733:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829210.854740:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829210.854833:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829210.854840:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829210.854845:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829211.878766:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829211.878772:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829211.878849:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829211.878856:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829211.878862:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829212.902756:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829212.902764:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829212.902863:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829212.902870:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829212.902874:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829213.926721:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829213.926727:0:815:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1787829213.926729:0:815:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1787829213.926731:0:815:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829213.926736:0:815:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1787829213.926738:0:815:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829213.926740:0:815:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829213.926742:0:815:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829213.926823:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829213.926831:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829213.926836:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829214.950745:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829214.950750:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829214.950805:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829214.950810:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829214.950815:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787829215.078762:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829215.078773:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1787829215.078791:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1787829215.078796:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787829215.078801:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1787829215.078809:0:468:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 4015 00000100:00080000:1.0:1787829215.078832:0:468:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1787829215.078835:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787829215.078839:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:1.0:1787829215.078843:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:1.0:1787829215.078846:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:1.0:1787829215.078850:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:1.0:1787829215.078868:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:1.0:1787829215.078870:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787829215.078872:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:1.0:1787829215.078878:0:468:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:1.0:1787829215.078881:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829215.078884:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1787829215.078887:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829215.078890:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829215.078892:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:1.0:1787829215.078895:0:468:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829215.078910:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1787829215.078912:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829215.078915:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1787829215.078935:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1787829215.078941:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829215.078959:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829215.078973:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d13ecf00 x1874670646807424/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787829215.079002:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787829215.079007:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829215.079018:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1787829215.079020:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829215.079023:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1787829215.079024:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1787829215.079025:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787829215.079027:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1787829215.079029:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4015 00000100:00080000:1.0:1787829215.079031:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1787829215.079033:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787829215.079034:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:1.0:1787829215.079036:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:1.0:1787829215.079037:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:1.0:1787829215.079038:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000020:00000001:1.0:1787829215.079040:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:1.0:1787829215.079041:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787829215.079042:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:1.0:1787829215.079044:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:1.0:1787829215.079046:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829215.079047:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1787829215.079064:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829215.079065:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829215.079067:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:1.0:1787829215.079072:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00000001:3.0:1787829215.079073:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000008:00000001:1.0:1787829215.079074:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829215.079078:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829215.079080:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 02000000:00000001:1.0:1787829215.079084:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:3.0:1787829215.079084:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 02000000:00000001:1.0:1787829215.079087:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829215.079089:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787829215.079090:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1787829215.079093:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:3.0:1787829215.079093:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829215.079095:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829215.079100:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:1.0:1787829215.079101:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829215.079104:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00100000:1.0:1787829215.079108:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d13eda40 x1874670646807552/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:3.0:1787829215.079109:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829215.079129:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d13ecf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646807424:192.168.203.143@tcp:250:kworker.0 00000100:00000001:1.0:1787829215.079131:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787829215.079135:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829215.079137:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:1.0:1787829215.079139:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1787829215.079141:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 02000000:00000001:3.0:1787829215.079143:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000100:00080000:1.0:1787829215.079145:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 02000000:00000001:3.0:1787829215.079147:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829215.079148:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1787829215.079150:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787829215.079154:0:468:0:(import.c:485:import_select_connection()) Process entered 02000000:00000001:3.0:1787829215.079154:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00080000:1.0:1787829215.079159:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4015 00000100:00080000:1.0:1787829215.079161:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1787829215.079162:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787829215.079164:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 02000000:00000001:3.0:1787829215.079164:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829215.079169:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:1.0:1787829215.079171:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:1.0:1787829215.079172:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000020:00000001:1.0:1787829215.079173:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:1.0:1787829215.079175:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787829215.079176:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:1.0:1787829215.079178:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:1.0:1787829215.079180:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829215.079181:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1787829215.079184:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829215.079194:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829215.079196:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:1.0:1787829215.079197:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1787829215.079198:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829215.079199:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829215.079200:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 02000000:00000001:1.0:1787829215.079203:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:3.0:1787829215.079205:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839541341952 : -126234168209664 : ffff8d30d13ecf00) 02000000:00000001:1.0:1787829215.079206:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829215.079209:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1787829215.079211:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1787829215.079213:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829215.079214:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:1.0:1787829215.079216:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829215.079221:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d13ee1c0 x1874670646807680/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787829215.079226:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787829215.079227:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829215.079229:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1787829215.079231:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829215.079234:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1787829215.079235:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1787829215.079236:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787829215.079238:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1787829215.079240:0:468:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4015 00000100:00080000:1.0:1787829215.079242:0:468:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1787829215.079244:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787829215.079245:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:1.0:1787829215.079246:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:1.0:1787829215.079247:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:1.0:1787829215.079260:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000020:00000001:1.0:1787829215.079261:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:1.0:1787829215.079262:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787829215.079263:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:1.0:1787829215.079265:0:468:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:1.0:1787829215.079266:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829215.079267:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1787829215.079269:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829215.079270:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829215.079272:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000800:00000100:3.0:1787829215.079275:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000008:00100000:1.0:1787829215.079276:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000008:00000001:1.0:1787829215.079280:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829215.079281:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829215.079284:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1787829215.079285:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829215.079286:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1787829215.079287:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1787829215.079288:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829215.079292:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829215.079295:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00100000:1.0:1787829215.079296:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d13ee580 x1874670646807808/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787829215.079301:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829215.079305:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829215.079309:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787829215.079313:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787829215.079315:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829215.079316:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829215.079318:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829215.079321:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4298689280 00000100:00000001:3.0:1787829215.079333:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 00000100:00000001:3.0:1787829215.079346:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4298689280 00000100:00000001:3.0:1787829215.079350:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 00000100:00000001:3.0:1787829215.079353:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829215.079355:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829215.079358:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1787829215.079364:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829215/real 1787829215] req@ffff8d30d13ecf00 x1874670646807424/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829240 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1787829215.079374:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787829215.079376:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829215.079379:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829215.079382:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829215.079383:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829215.079386:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829215.079388:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829215.079393:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d13ecf00 x1874670646807424/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829240 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829215.079402:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829215.079406:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829215.079409:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829215.079411:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829215.079412:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829215.079415:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829215.079417:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829215.079421:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829215.079427:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829215.079430:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829215.079434:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d13ecf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646807424:192.168.203.143@tcp:250:kworker.0 00000100:00000001:3.0:1787829215.079440:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829215.079443:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829215.079445:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829215.079448:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829215.079452:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829215.079465:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829215.079468:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829215.079473:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829215.079475:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829215.079479:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829215.079480:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829215.079482:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829215.079517:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787829215.079530:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829215.079535:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1787829215.079536:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787829215.079538:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787829215.079539:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787829215.079541:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829215.079543:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787829215.079545:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829215.079546:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829215.079549:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d13eda40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646807552:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829215.079552:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1787829215.079554:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1787829215.079556:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829215.079558:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1787829215.079560:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829215.079566:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787829215.079568:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839541344832 : -126234168206784 : ffff8d30d13eda40) 00000100:00000001:3.0:1787829215.079573:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1787829215.079585:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:3.0:1787829215.079590:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787829215.079592:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829215.079593:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829215.079594:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1787829215.079597:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829215.079653:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298688023 00000100:00000001:3.0:1787829215.079657:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298689280 00000100:00000001:3.0:1787829215.079659:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298688023 00000100:00000001:3.0:1787829215.079660:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298689280 00000100:00000001:3.0:1787829215.079662:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829215.079663:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829215.079664:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1787829215.079668:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829215/real 1787829215] req@ffff8d30d13eda40 x1874670646807552/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829240 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1787829215.079674:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787829215.079675:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829215.079677:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829215.079679:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829215.079680:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829215.079681:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829215.079682:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829215.079685:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d13eda40 x1874670646807552/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829240 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829215.079690:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829215.079691:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829215.079693:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829215.079694:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829215.079695:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829215.079696:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829215.079697:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829215.079699:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829215.079701:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829215.079702:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829215.079704:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d13eda40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646807552:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829215.079706:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787829215.079707:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787829215.079708:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829215.079709:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787829215.079710:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829215.079712:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829215.079713:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d13ee1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646807680:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829215.079716:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1787829215.079717:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1787829215.079718:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829215.079719:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1787829215.079722:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829215.079725:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787829215.079726:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839541346752 : -126234168204864 : ffff8d30d13ee1c0) 00000100:00000001:3.0:1787829215.079728:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1787829215.079737:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:3.0:1787829215.079741:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787829215.079742:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829215.079743:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829215.079744:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1787829215.079746:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829215.079749:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298688023 00000100:00000001:3.0:1787829215.079751:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298689280 00000100:00000001:3.0:1787829215.079752:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298688023 00000100:00000001:3.0:1787829215.079769:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298689280 00000100:00000001:3.0:1787829215.079771:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829215.079772:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829215.079773:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1787829215.079776:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829215/real 1787829215] req@ffff8d30d13ee1c0 x1874670646807680/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829240 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1787829215.079781:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787829215.079781:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829215.079783:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829215.079784:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829215.079785:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829215.079786:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829215.079786:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829215.079789:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d13ee1c0 x1874670646807680/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829240 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829215.079794:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829215.079795:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829215.079808:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829215.079809:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829215.079809:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829215.079810:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829215.079811:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829215.079813:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829215.079815:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829215.079817:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829215.079819:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d13ee1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646807680:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829215.079822:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787829215.079839:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787829215.079840:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829215.079842:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787829215.079843:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829215.079844:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829215.079846:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d13ee580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646807808:192.168.203.143@tcp:8:kworker.0 00000100:00000001:3.0:1787829215.079848:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1787829215.079850:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1787829215.079851:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829215.079852:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1787829215.079854:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829215.079858:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787829215.079859:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839541347712 : -126234168203904 : ffff8d30d13ee580) 00000100:00000001:3.0:1787829215.079862:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1787829215.079870:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:3.0:1787829215.079873:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787829215.079875:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829215.079876:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829215.079877:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1787829215.079879:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829215.079881:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298688036 00000100:00000001:3.0:1787829215.079884:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298689280 00000100:00000001:3.0:1787829215.079885:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298688036 00000100:00000001:3.0:1787829215.079887:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298689280 00000100:00000001:3.0:1787829215.079889:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829215.079891:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829215.079892:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1787829215.079895:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829215/real 1787829215] req@ffff8d30d13ee580 x1874670646807808/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829240 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1787829215.079902:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787829215.079903:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829215.079904:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829215.079906:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829215.079906:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829215.079908:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829215.079909:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829215.079912:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d13ee580 x1874670646807808/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829240 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829215.079917:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829215.079919:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829215.079920:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829215.079922:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829215.079922:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829215.079924:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829215.079925:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829215.079926:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829215.079928:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829215.079930:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829215.079932:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d13ee580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646807808:192.168.203.143@tcp:8:kworker.0 00000100:00000001:3.0:1787829215.079934:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829215.079936:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829215.079937:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829215.079938:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829215.079939:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829215.079941:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829215.079941:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829215.079943:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829215.079944:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829215.079946:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829215.079947:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829215.079949:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829215.079950:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829215.079951:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829215.079952:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829215.079953:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829215.079954:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829215.079955:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829215.079956:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829215.079958:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829215.079959:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829215.079960:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829215.079961:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829215.079962:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829215.079963:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829215.079964:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829215.079964:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829215.079966:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829215.079967:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829215.079968:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829215.079968:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829215.079970:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829215.079972:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787829215.079973:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829215.079975:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1787829215.079976:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829215.974780:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829215.974789:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829215.974887:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829215.974907:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829215.974913:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829216.998713:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829216.998720:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829216.998800:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829216.998807:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829216.998812:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829218.022701:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829218.022707:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829218.022769:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829218.022774:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829218.022778:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829219.046689:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829219.046696:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829219.046743:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829219.046748:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829219.046751:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829220.070681:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829220.070686:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829220.070729:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829220.070733:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829220.070736:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787829220.198681:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829220.198689:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1787829220.198691:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1787829220.198695:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787829220.198698:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1787829220.198704:0:468:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 4020 00000100:00080000:1.0:1787829220.198714:0:468:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1787829220.198717:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787829220.198719:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:1.0:1787829220.198723:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:1.0:1787829220.198725:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:1.0:1787829220.198729:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:1.0:1787829220.198732:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:1.0:1787829220.198734:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787829220.198735:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:1.0:1787829220.198740:0:468:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:1.0:1787829220.198743:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829220.198746:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1787829220.198750:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829220.198753:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829220.198756:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:1.0:1787829220.198759:0:468:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829220.198770:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1787829220.198773:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829220.198776:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1787829220.198777:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1787829220.198781:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829220.198791:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829220.198799:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d13ef480 x1874670646807936/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787829220.198807:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787829220.198810:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829220.198825:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1787829220.198827:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829220.198833:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1787829220.198834:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1787829220.198836:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787829220.198838:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1787829220.198840:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4020 00000100:00080000:1.0:1787829220.198843:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1787829220.198844:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787829220.198845:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:1.0:1787829220.198847:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:1.0:1787829220.198848:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:1.0:1787829220.198850:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000020:00000001:1.0:1787829220.198852:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:1.0:1787829220.198853:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787829220.198854:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:1.0:1787829220.198857:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:3.0:1787829220.198857:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787829220.198861:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829220.198862:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 02000000:00000001:1.0:1787829220.198863:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:3.0:1787829220.198867:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 02000000:00000001:1.0:1787829220.198870:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829220.198871:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 02000000:00000001:1.0:1787829220.198875:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829220.198875:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829220.198879:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 02000000:00000001:3.0:1787829220.198881:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000008:00100000:1.0:1787829220.198883:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 02000000:00000001:3.0:1787829220.198885:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000008:00000001:1.0:1787829220.198886:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829220.198891:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829220.198891:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829220.198898:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00100000:3.0:1787829220.198899:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d13ef480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646807936:192.168.203.143@tcp:250:kworker.0 02000000:00000001:1.0:1787829220.198900:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829220.198904:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1787829220.198906:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:3.0:1787829220.198908:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787829220.198909:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829220.198912:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:1.0:1787829220.198915:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829220.198915:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829220.198920:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00100000:1.0:1787829220.198923:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d13ef840 x1874670646808064/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:3.0:1787829220.198925:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829220.198933:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787829220.198934:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829220.198937:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1787829220.198939:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829220.198941:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:1.0:1787829220.198943:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:3.0:1787829220.198943:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00080000:1.0:1787829220.198948:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787829220.198948:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839541351552 : -126234168200064 : ffff8d30d13ef480) 00000100:00000001:1.0:1787829220.198950:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1787829220.198953:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4020 00000100:00000001:3.0:1787829220.198956:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00080000:1.0:1787829220.198957:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1787829220.198960:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787829220.198961:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:1.0:1787829220.198963:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:1.0:1787829220.198964:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:1.0:1787829220.198965:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000020:00000001:1.0:1787829220.198967:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:1.0:1787829220.198968:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787829220.198969:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:1.0:1787829220.198971:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:1.0:1787829220.198973:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829220.198975:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1787829220.198977:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829220.198978:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829220.198979:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:1.0:1787829220.198981:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1787829220.198983:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829220.198984:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829220.198987:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1787829220.198988:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829220.198989:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1787829220.198990:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1787829220.198991:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829220.198994:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829220.198999:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d13eed00 x1874670646808192/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1787829220.199000:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787829220.199007:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787829220.199009:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829220.199011:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1787829220.199013:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1787829220.199014:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00080000:1.0:1787829220.199016:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1787829220.199019:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:3.0:1787829220.199020:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00080000:1.0:1787829220.199021:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787829220.199025:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829220.199026:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1787829220.199030:0:468:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4020 00000100:00000001:3.0:1787829220.199031:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1787829220.199037:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1787829220.199038:0:468:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1787829220.199041:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829220.199041:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4298694400 00000100:00000001:1.0:1787829220.199042:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:1.0:1787829220.199047:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:1.0:1787829220.199049:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:1.0:1787829220.199050:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000020:00000001:1.0:1787829220.199051:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:1.0:1787829220.199053:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829220.199054:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 00000100:00000001:1.0:1787829220.199055:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:1.0:1787829220.199059:0:468:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:1.0:1787829220.199062:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829220.199063:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1787829220.199065:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829220.199066:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829220.199067:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:3.0:1787829220.199068:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4298694400 00000008:00100000:1.0:1787829220.199069:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000008:00000001:1.0:1787829220.199074:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829220.199075:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829220.199076:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 02000000:00000001:1.0:1787829220.199079:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:3.0:1787829220.199080:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829220.199081:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829220.199085:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:3.0:1787829220.199085:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829220.199088:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1787829220.199090:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829220.199090:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00000001:1.0:1787829220.199096:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829220.199098:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829220/real 1787829220] req@ffff8d30d13ef480 x1874670646807936/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829245 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829220.199102:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d13ede00 x1874670646808320/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000001:3.0:1787829220.199120:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000001:1.0:1787829220.199121:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787829220.199125:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:3.0:1787829220.199125:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787829220.199129:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829220.199130:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829220.199134:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829220.199136:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829220.199138:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829220.199139:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829220.199144:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d13ef480 x1874670646807936/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829245 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829220.199151:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829220.199154:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829220.199158:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829220.199160:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829220.199161:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829220.199164:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829220.199168:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829220.199171:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829220.199176:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829220.199179:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829220.199185:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d13ef480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646807936:192.168.203.143@tcp:250:kworker.0 00000100:00000001:3.0:1787829220.199191:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829220.199193:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829220.199194:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829220.199197:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829220.199200:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829220.199203:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829220.199206:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829220.199211:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829220.199213:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829220.199217:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829220.199218:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829220.199221:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829220.199227:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787829220.199229:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829220.199232:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1787829220.199233:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787829220.199235:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787829220.199237:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787829220.199240:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829220.199243:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787829220.199245:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829220.199247:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829220.199252:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d13ef840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646808064:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829220.199256:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1787829220.199258:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1787829220.199260:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829220.199261:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1787829220.199264:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829220.199270:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787829220.199272:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839541352512 : -126234168199104 : ffff8d30d13ef840) 00000100:00000001:3.0:1787829220.199277:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1787829220.199290:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:3.0:1787829220.199294:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787829220.199296:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829220.199297:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829220.199298:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1787829220.199301:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829220.199304:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298693142 00000100:00000001:3.0:1787829220.199306:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298694400 00000100:00000001:3.0:1787829220.199309:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298693142 00000100:00000001:3.0:1787829220.199310:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298694400 00000100:00000001:3.0:1787829220.199312:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829220.199314:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829220.199315:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1787829220.199318:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829220/real 1787829220] req@ffff8d30d13ef840 x1874670646808064/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829245 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1787829220.199324:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787829220.199325:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829220.199328:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829220.199329:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829220.199330:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829220.199331:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829220.199332:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829220.199335:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d13ef840 x1874670646808064/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829245 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829220.199345:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829220.199347:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829220.199350:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829220.199351:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829220.199352:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829220.199353:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829220.199354:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829220.199356:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829220.199358:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829220.199420:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829220.199425:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d13ef840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646808064:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829220.199431:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787829220.199433:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787829220.199435:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829220.199437:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787829220.199439:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829220.199442:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829220.199446:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d13eed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646808192:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829220.199450:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1787829220.199453:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1787829220.199551:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829220.199554:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1787829220.199557:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829220.199561:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787829220.199562:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839541349632 : -126234168201984 : ffff8d30d13eed00) 00000100:00000001:3.0:1787829220.199578:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1787829220.199593:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:3.0:1787829220.199632:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787829220.199636:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829220.199638:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829220.199641:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1787829220.199646:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829220.199650:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298693143 00000100:00000001:3.0:1787829220.199653:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298694400 00000100:00000001:3.0:1787829220.199656:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298693143 00000100:00000001:3.0:1787829220.199658:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298694400 00000100:00000001:3.0:1787829220.199660:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829220.199662:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829220.199664:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1787829220.199717:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829220/real 1787829220] req@ffff8d30d13eed00 x1874670646808192/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829245 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1787829220.199726:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787829220.199728:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829220.199731:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829220.199733:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829220.199734:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829220.199736:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829220.199738:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829220.199742:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d13eed00 x1874670646808192/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829245 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829220.199750:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829220.199753:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829220.199755:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829220.199757:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829220.199758:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829220.199761:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829220.199762:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829220.199766:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829220.199769:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829220.199772:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829220.199776:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d13eed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646808192:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829220.199780:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787829220.199782:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787829220.199783:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829220.199786:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787829220.199788:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829220.199790:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829220.199793:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d13ede00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646808320:192.168.203.143@tcp:8:kworker.0 00000100:00000001:3.0:1787829220.199797:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1787829220.199799:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1787829220.199801:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829220.199803:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1787829220.199806:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829220.199813:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787829220.199814:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839541345792 : -126234168205824 : ffff8d30d13ede00) 00000100:00000001:3.0:1787829220.199818:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1787829220.199833:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:3.0:1787829220.199838:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787829220.199841:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829220.199842:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829220.199844:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1787829220.199847:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829220.199851:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298693156 00000100:00000001:3.0:1787829220.199853:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298694400 00000100:00000001:3.0:1787829220.199855:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298693156 00000100:00000001:3.0:1787829220.199858:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298694400 00000100:00000001:3.0:1787829220.199860:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829220.199862:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829220.199864:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1787829220.199869:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829220/real 1787829220] req@ffff8d30d13ede00 x1874670646808320/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829245 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1787829220.199885:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787829220.199886:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829220.199889:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829220.199891:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829220.199892:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829220.199894:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829220.199896:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829220.199900:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d13ede00 x1874670646808320/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829245 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829220.199908:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829220.199910:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829220.199913:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829220.199914:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829220.199916:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829220.199918:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829220.199920:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829220.199922:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829220.199926:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829220.199928:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829220.199932:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d13ede00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646808320:192.168.203.143@tcp:8:kworker.0 00000100:00000001:3.0:1787829220.199936:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829220.199938:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829220.199939:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829220.199941:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829220.199944:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829220.199945:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829220.199946:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829220.199949:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829220.199950:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829220.199953:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829220.199954:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829220.199956:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829220.199958:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829220.199959:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829220.199961:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829220.199962:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829220.199964:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829220.199966:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829220.199967:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829220.199969:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829220.199971:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829220.199972:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829220.199974:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829220.199975:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829220.199977:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829220.199978:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829220.199979:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829220.199980:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829220.199982:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829220.199984:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829220.199985:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829220.199987:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829220.199991:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787829220.199992:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829220.199994:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1787829220.199996:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829221.094692:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829221.094698:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829221.094752:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829221.094756:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829221.094761:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829222.118694:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829222.118700:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829222.118780:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829222.118786:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829222.118790:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829223.142690:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829223.142695:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829223.142742:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829223.142745:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829223.142749:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829224.166683:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829224.166689:0:815:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1787829224.166691:0:815:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1787829224.166694:0:815:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829224.166698:0:815:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1787829224.166700:0:815:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829224.166701:0:815:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829224.166703:0:815:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829224.166776:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829224.166781:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829224.166786:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829225.190724:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829225.190731:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829225.190792:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829225.190795:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829225.190798:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787829225.318692:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829225.318703:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1787829225.318706:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1787829225.318709:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787829225.318714:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1787829225.318722:0:468:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 4025 00000100:00080000:1.0:1787829225.318739:0:468:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1787829225.318743:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787829225.318745:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:1.0:1787829225.318748:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:1.0:1787829225.318751:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:1.0:1787829225.318754:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:1.0:1787829225.318756:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:1.0:1787829225.318758:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787829225.318759:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:1.0:1787829225.318763:0:468:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:1.0:1787829225.318764:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829225.318767:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1787829225.318770:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829225.318772:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829225.318774:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:1.0:1787829225.318777:0:468:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829225.318791:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1787829225.318792:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829225.318795:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1787829225.318798:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1787829225.318803:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829225.318815:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829225.318824:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d13ef0c0 x1874670646808448/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787829225.318833:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787829225.318837:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829225.318856:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1787829225.318858:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829225.318860:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1787829225.318862:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1787829225.318863:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787829225.318865:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1787829225.318866:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4025 00000100:00080000:1.0:1787829225.318869:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1787829225.318870:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787829225.318871:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:1.0:1787829225.318872:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:1.0:1787829225.318873:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:1.0:1787829225.318875:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000020:00000001:1.0:1787829225.318876:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:1.0:1787829225.318878:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787829225.318879:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:1.0:1787829225.318881:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:1.0:1787829225.318882:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829225.318884:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1787829225.318889:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829225.318891:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829225.318893:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:1.0:1787829225.318897:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1787829225.318899:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829225.318901:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829225.318904:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1787829225.318905:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829225.318906:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1787829225.318907:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:3.0:1787829225.318908:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:1.0:1787829225.318909:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829225.318915:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1787829225.318916:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829225.318919:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00100000:1.0:1787829225.318923:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d13efc00 x1874670646808576/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787829225.318924:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787829225.318928:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829225.318934:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:1.0:1787829225.318935:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 02000000:00000001:3.0:1787829225.318937:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829225.318938:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829225.318941:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829225.318943:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1787829225.318947:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1787829225.318948:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d13ef0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646808448:192.168.203.143@tcp:250:kworker.0 00000100:00080000:1.0:1787829225.318953:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1787829225.318955:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:1.0:1787829225.318956:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1787829225.318958:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787829225.318960:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1787829225.318962:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4025 00000100:00080000:1.0:1787829225.318964:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1787829225.318965:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787829225.318967:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:1.0:1787829225.318969:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:1.0:1787829225.318970:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:1.0:1787829225.318971:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 02000000:00000001:3.0:1787829225.318971:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000020:00000001:1.0:1787829225.318975:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 02000000:00000001:3.0:1787829225.318975:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829225.318976:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787829225.318979:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 02000000:00000001:3.0:1787829225.318979:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00080000:1.0:1787829225.318983:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:1.0:1787829225.318985:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829225.318987:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787829225.318987:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829225.318990:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829225.318993:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829225.318994:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:1.0:1787829225.318995:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1787829225.318997:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829225.318998:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829225.319002:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1787829225.319002:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829225.319003:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1787829225.319004:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1787829225.319005:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829225.319009:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829225.319010:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00100000:1.0:1787829225.319013:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d13ed2c0 x1874670646808704/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787829225.319014:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839541350592 : -126234168201024 : ffff8d30d13ef0c0) 00000100:00000001:3.0:1787829225.319022:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:1.0:1787829225.319023:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787829225.319026:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829225.319028:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1787829225.319030:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829225.319032:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1787829225.319033:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1787829225.319034:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787829225.319035:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1787829225.319037:0:468:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4025 00000100:00080000:1.0:1787829225.319039:0:468:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1787829225.319040:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787829225.319041:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:1.0:1787829225.319042:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:1.0:1787829225.319043:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:1.0:1787829225.319044:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000020:00000001:1.0:1787829225.319045:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:1.0:1787829225.319046:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787829225.319047:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:1.0:1787829225.319049:0:468:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:1.0:1787829225.319051:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829225.319053:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1787829225.319055:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829225.319056:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829225.319057:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:1.0:1787829225.319059:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000008:00000001:1.0:1787829225.319060:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829225.319061:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829225.319064:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1787829225.319065:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829225.319066:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1787829225.319067:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1787829225.319068:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829225.319071:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829225.319073:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d13ed680 x1874670646808832/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787829225.319078:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787829225.319080:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829225.319082:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829225.319099:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829225.319105:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4298699520 00000100:00000001:3.0:1787829225.319116:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 00000100:00000001:3.0:1787829225.319126:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4298699520 00000100:00000001:3.0:1787829225.319132:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 00000100:00000001:3.0:1787829225.319136:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829225.319140:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829225.319145:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829225.319148:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829225.319159:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787829225.319162:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:3.0:1787829225.319165:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1787829225.319168:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787829225.319170:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787829225.319173:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787829225.319175:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829225.319178:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787829225.319180:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829225.319183:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829225.319187:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d13efc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646808576:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829225.319190:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1787829225.319193:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1787829225.319194:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829225.319195:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1787829225.319198:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829225.319204:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787829225.319205:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839541353472 : -126234168198144 : ffff8d30d13efc00) 00000100:00000001:3.0:1787829225.319210:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:3.0:1787829225.319222:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829225.319226:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298698262 00000100:00000001:3.0:1787829225.319228:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298699520 00000100:00000001:3.0:1787829225.319230:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298698262 00000100:00000001:3.0:1787829225.319232:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298699520 00000100:00000001:3.0:1787829225.319234:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829225.319235:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829225.319236:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787829225.319237:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787829225.319238:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829225.319239:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787829225.319240:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829225.319242:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829225.319244:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d13ed2c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646808704:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829225.319247:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1787829225.319249:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1787829225.319250:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829225.319251:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1787829225.319253:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829225.319257:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787829225.319258:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839541342912 : -126234168208704 : ffff8d30d13ed2c0) 00000100:00000001:3.0:1787829225.319260:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:3.0:1787829225.319266:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829225.319268:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298698262 00000100:00000001:3.0:1787829225.319270:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298699520 00000100:00000001:3.0:1787829225.319272:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298698262 00000100:00000001:3.0:1787829225.319274:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298699520 00000100:00000001:3.0:1787829225.319276:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829225.319277:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829225.319278:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787829225.319280:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787829225.319282:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829225.319283:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787829225.319285:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829225.319286:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829225.319288:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d13ed680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646808832:192.168.203.143@tcp:8:kworker.0 00000100:00000001:3.0:1787829225.319289:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1787829225.319290:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1787829225.319291:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829225.319293:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1787829225.319295:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829225.319298:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787829225.319299:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839541343872 : -126234168207744 : ffff8d30d13ed680) 00000100:00000001:3.0:1787829225.319301:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:3.0:1787829225.319306:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829225.319308:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298698275 00000100:00000001:3.0:1787829225.319310:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298699520 00000100:00000001:3.0:1787829225.319312:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298698275 00000100:00000001:3.0:1787829225.319313:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298699520 00000100:00000001:3.0:1787829225.319316:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829225.319318:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829225.319321:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829225.319322:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829225.319325:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787829225.319326:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:3.0:1787829225.319328:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1787829225.319329:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787829225.319333:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829225.319335:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829226.214722:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829226.214731:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829226.214782:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829226.214787:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829226.214791:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829227.238672:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829227.238679:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829227.238726:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829227.238729:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829227.238732:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829228.262716:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829228.262725:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829228.262785:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829228.262792:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829228.262797:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:2.0:1787829228.390907:0:2381:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.143:988 00000400:00000100:2.0:1787829228.390967:0:2381:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.143@tcp at host 192.168.203.143 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1787829228.390989:0:2381:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.143@tcp 00000100:00000001:2.0:1787829228.391007:0:2381:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1787829228.391028:0:2381:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787829228.391031:0:2381:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829228.391032:0:2381:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:2.0:1787829228.391041:0:2381:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.143@tcp 00000100:00000001:2.0:1787829228.391045:0:2381:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1787829228.391047:0:2381:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787829228.391048:0:2381:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829228.391049:0:2381:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:2.0:1787829228.391052:0:2381:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.143@tcp 00000100:00000001:2.0:1787829228.391056:0:2381:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1787829228.391058:0:2381:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787829228.391059:0:2381:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829228.391060:0:2381:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:2.0:1787829228.391063:0:2381:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.143@tcp 00000100:00000001:2.0:1787829228.391066:0:2381:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1787829228.391068:0:2381:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787829228.391069:0:2381:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829228.391070:0:2381:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1787829228.391144:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1787829228.391153:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787829228.391157:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1787829228.391173:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829225/real 1787829228] req@ffff8d30d13efc00 x1874670646808576/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829250 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1787829228.391191:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787829228.391194:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829228.391204:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829228.391207:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829228.391209:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829228.391213:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829228.391215:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829228.391220:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d13efc00 x1874670646808576/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829250 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829228.391228:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829228.391373:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829228.391377:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829228.391380:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829228.391381:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829228.391384:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829228.391386:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829228.391391:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829228.391397:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829228.391400:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829228.391404:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d13efc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646808576:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829228.391410:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1787829228.391416:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829225/real 1787829228] req@ffff8d30d13ed2c0 x1874670646808704/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829250 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1787829228.391426:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787829228.391427:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829228.391431:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829228.391433:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829228.391434:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829228.391437:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829228.391438:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829228.391443:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d13ed2c0 x1874670646808704/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829250 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829228.391450:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829228.391452:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829228.391455:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829228.391457:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829228.391458:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829228.391460:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829228.391461:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829228.391464:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829228.391468:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829228.391470:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829228.391474:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d13ed2c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646808704:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829228.391478:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1787829228.391483:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829225/real 1787829228] req@ffff8d30d13ed680 x1874670646808832/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829250 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1787829228.391492:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787829228.391493:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829228.391496:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829228.391497:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829228.391498:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829228.391500:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829228.391501:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829228.391505:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d13ed680 x1874670646808832/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829250 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829228.391527:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829228.391530:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829228.391532:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829228.391534:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829228.391535:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829228.391537:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829228.391539:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829228.391542:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829228.391546:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829228.391548:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829228.391552:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d13ed680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646808832:192.168.203.143@tcp:8:kworker.0 00000100:00000001:3.0:1787829228.391556:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1787829228.391561:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829225/real 1787829228] req@ffff8d30d13ef0c0 x1874670646808448/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829250 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1787829228.391570:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787829228.391571:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829228.391574:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829228.391576:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829228.391577:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829228.391579:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829228.391580:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829228.391584:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d13ef0c0 x1874670646808448/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829250 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829228.391592:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829228.391594:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829228.391597:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829228.391599:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829228.391624:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829228.391626:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829228.391628:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829228.391632:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829228.391637:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829228.391640:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829228.391646:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d13ef0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646808448:192.168.203.143@tcp:250:kworker.0 00000100:00000001:3.0:1787829228.391651:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829228.391655:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829228.391658:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829228.391662:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829228.391668:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829228.391671:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829228.391674:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829228.391679:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829228.391682:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829228.391687:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829228.391689:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829228.391692:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829228.391693:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829228.391695:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829228.391697:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829228.391699:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829228.391701:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829228.391704:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829228.391705:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829228.391707:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829228.391708:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829228.391711:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829228.391712:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829228.391713:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829228.391822:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829228.391824:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829228.391825:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829228.391828:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829228.391829:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829228.391831:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829228.391833:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829228.391835:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829228.391837:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829228.391838:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829228.391841:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829228.391842:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829228.391844:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829228.391845:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829228.391846:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829228.391848:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829228.391849:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829228.391852:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829228.391859:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787829228.391861:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829228.391864:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1787829228.391866:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829229.286709:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829229.286716:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829229.286795:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829229.286801:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829229.286805:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829230.310725:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829230.310730:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829230.310780:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829230.310784:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829230.310786:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787829230.438721:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829230.438732:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1787829230.438735:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787829230.438739:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787829230.438743:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787829230.438751:0:468:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 4030 00000100:00080000:2.0:1787829230.438767:0:468:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787829230.438769:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829230.438771:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787829230.438774:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787829230.438776:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787829230.438778:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:2.0:1787829230.438781:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:2.0:1787829230.438782:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829230.438783:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:2.0:1787829230.438787:0:468:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787829230.438789:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829230.438791:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787829230.438793:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787829230.438795:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829230.438797:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:2.0:1787829230.438799:0:468:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829230.438806:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787829230.438807:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829230.438810:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787829230.438812:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787829230.438815:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829230.438824:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787829230.438831:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2fc00 x1874670646808960/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787829230.438839:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787829230.438843:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829230.438859:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787829230.438862:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829230.438865:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1787829230.438867:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787829230.438869:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787829230.438872:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787829230.438875:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4030 00000100:00080000:2.0:1787829230.438879:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787829230.438881:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829230.438883:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787829230.438885:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787829230.438887:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787829230.438889:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000020:00000001:2.0:1787829230.438891:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:2.0:1787829230.438893:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829230.438895:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:2.0:1787829230.438898:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787829230.438901:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829230.438903:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787829230.438909:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787829230.438911:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829230.438913:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1787829230.438917:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1787829230.438929:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829230.438931:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829230.438936:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787829230.438937:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829230.438940:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787829230.438941:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787829230.438944:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829230.438950:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787829230.438955:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2c3c0 x1874670646809088/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787829230.438964:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787829230.438967:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829230.438969:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787829230.438972:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829230.438976:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1787829230.438978:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787829230.438979:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787829230.438982:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787829230.438985:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4030 00000100:00080000:2.0:1787829230.438989:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787829230.438992:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829230.438994:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787829230.438996:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787829230.438998:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787829230.438999:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000020:00000001:2.0:1787829230.439007:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:2.0:1787829230.439013:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829230.439015:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:2.0:1787829230.439019:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787829230.439022:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829230.439024:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787829230.439027:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787829230.439030:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829230.439032:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1787829230.439034:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1787829230.439037:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829230.439039:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829230.439044:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787829230.439045:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829230.439046:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787829230.439048:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787829230.439050:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829230.439056:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787829230.439061:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2de00 x1874670646809216/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787829230.439070:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787829230.439072:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829230.439075:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787829230.439078:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829230.439082:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1787829230.439083:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787829230.439085:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787829230.439088:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787829230.439091:0:468:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4030 00000100:00080000:2.0:1787829230.439095:0:468:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787829230.439098:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829230.439099:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787829230.439101:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787829230.439103:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787829230.439105:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000020:00000001:2.0:1787829230.439107:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:2.0:1787829230.439110:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829230.439111:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:2.0:1787829230.439115:0:468:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787829230.439117:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829230.439119:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787829230.439122:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787829230.439124:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829230.439126:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1787829230.439128:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000008:00000001:2.0:1787829230.439130:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829230.439132:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829230.439136:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787829230.439138:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829230.439139:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787829230.439141:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787829230.439143:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829230.439147:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787829230.439153:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2c780 x1874670646809344/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787829230.439160:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787829230.439163:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829230.439165:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829230.439306:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1787829230.439313:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787829230.439316:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787829230.439319:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787829230.439322:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829230.439338:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787829230.439342:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829230.439345:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829230.439351:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646808960:192.168.203.143@tcp:250:kworker.0 00000100:00000001:3.0:1787829230.439357:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1787829230.439361:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1787829230.439364:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829230.439367:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1787829230.439372:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829230.439387:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787829230.439389:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417883648 : -126234291667968 : ffff8d30c9e2fc00) 00000100:00000001:3.0:1787829230.439393:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1787829230.439431:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:3.0:1787829230.439444:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787829230.439448:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829230.439451:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829230.439453:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1787829230.439460:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829230.439464:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4298704640 00000100:00000001:3.0:1787829230.439559:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 00000100:00000001:3.0:1787829230.439567:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4298704640 00000100:00000001:3.0:1787829230.439571:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 00000100:00000001:3.0:1787829230.439575:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829230.439578:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829230.439581:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1787829230.439587:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829230/real 1787829230] req@ffff8d30c9e2fc00 x1874670646808960/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829255 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1787829230.439622:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787829230.439625:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829230.439629:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829230.439631:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829230.439633:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829230.439635:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829230.439637:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829230.439643:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2fc00 x1874670646808960/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829255 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829230.439652:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829230.439655:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829230.439658:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829230.439660:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829230.439661:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829230.439664:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829230.439666:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829230.439669:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829230.439673:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829230.439676:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829230.439680:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646808960:192.168.203.143@tcp:250:kworker.0 00000100:00000001:3.0:1787829230.439685:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787829230.439687:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787829230.439689:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829230.439691:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787829230.439693:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829230.439696:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829230.439699:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646809088:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829230.439703:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1787829230.439705:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1787829230.439706:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829230.439708:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1787829230.439712:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829230.439718:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787829230.439719:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417869248 : -126234291682368 : ffff8d30c9e2c3c0) 00000100:00000001:3.0:1787829230.439724:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1787829230.439737:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:3.0:1787829230.439741:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787829230.439744:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829230.439745:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829230.439747:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1787829230.439750:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829230.439754:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298703383 00000100:00000001:3.0:1787829230.439757:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298704640 00000100:00000001:3.0:1787829230.439760:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298703383 00000100:00000001:3.0:1787829230.439762:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298704640 00000100:00000001:3.0:1787829230.439764:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829230.439767:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829230.439769:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1787829230.439773:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829230/real 1787829230] req@ffff8d30c9e2c3c0 x1874670646809088/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829255 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1787829230.439789:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787829230.439790:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829230.439793:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829230.439794:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829230.439796:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829230.439798:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829230.439800:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829230.439814:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2c3c0 x1874670646809088/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829255 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829230.439823:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829230.439825:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829230.439827:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829230.439829:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829230.439831:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829230.439832:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829230.439834:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829230.439837:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829230.439841:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829230.439844:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829230.439847:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646809088:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829230.439852:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787829230.439853:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787829230.439855:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829230.439856:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787829230.439858:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829230.439860:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829230.439864:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646809216:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829230.439867:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1787829230.439869:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1787829230.439871:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829230.439873:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1787829230.439876:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829230.439880:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787829230.439882:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417875968 : -126234291675648 : ffff8d30c9e2de00) 00000100:00000001:3.0:1787829230.439886:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1787829230.439896:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:3.0:1787829230.439901:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787829230.439903:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829230.439905:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829230.439907:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1787829230.439909:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829230.439913:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298703383 00000100:00000001:3.0:1787829230.439916:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298704640 00000100:00000001:3.0:1787829230.439918:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298703383 00000100:00000001:3.0:1787829230.439921:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298704640 00000100:00000001:3.0:1787829230.439923:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829230.439925:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829230.439926:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1787829230.439931:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829230/real 1787829230] req@ffff8d30c9e2de00 x1874670646809216/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829255 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1787829230.439939:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787829230.439941:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829230.439943:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829230.439945:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829230.439946:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829230.439948:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829230.439949:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829230.439954:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2de00 x1874670646809216/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829255 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829230.439967:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829230.439970:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829230.439973:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829230.439975:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829230.439977:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829230.439978:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829230.439980:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829230.439983:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829230.439987:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829230.439989:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829230.439993:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646809216:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829230.439997:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787829230.439999:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787829230.440000:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829230.440003:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787829230.440005:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829230.440007:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829230.440010:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646809344:192.168.203.143@tcp:8:kworker.0 00000100:00000001:3.0:1787829230.440014:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1787829230.440015:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1787829230.440017:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829230.440019:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1787829230.440022:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829230.440027:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787829230.440029:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417870208 : -126234291681408 : ffff8d30c9e2c780) 00000100:00000001:3.0:1787829230.440033:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1787829230.440043:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:3.0:1787829230.440047:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787829230.440049:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829230.440051:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829230.440052:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1787829230.440055:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829230.440059:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298703396 00000100:00000001:3.0:1787829230.440061:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298704640 00000100:00000001:3.0:1787829230.440064:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298703396 00000100:00000001:3.0:1787829230.440067:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298704640 00000100:00000001:3.0:1787829230.440069:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829230.440071:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829230.440073:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1787829230.440077:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829230/real 1787829230] req@ffff8d30c9e2c780 x1874670646809344/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829255 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1787829230.440084:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787829230.440086:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829230.440088:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829230.440090:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829230.440091:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829230.440093:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829230.440095:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829230.440099:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2c780 x1874670646809344/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829255 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829230.440106:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829230.440108:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829230.440111:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829230.440113:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829230.440114:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829230.440115:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829230.440117:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829230.440119:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829230.440122:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829230.440125:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829230.440129:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646809344:192.168.203.143@tcp:8:kworker.0 00000100:00000001:3.0:1787829230.440133:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829230.440135:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829230.440137:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829230.440140:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829230.440143:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829230.440145:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829230.440147:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829230.440151:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829230.440153:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829230.440157:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829230.440158:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829230.440160:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829230.440161:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829230.440163:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829230.440164:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829230.440165:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829230.440167:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829230.440168:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829230.440169:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829230.440172:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829230.440173:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829230.440175:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829230.440176:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829230.440177:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829230.440179:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829230.440181:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829230.440182:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829230.440184:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829230.440185:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829230.440196:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829230.440198:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829230.440200:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829230.440201:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829230.440202:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829230.440204:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829230.440206:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829230.440207:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829230.440209:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829230.440210:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829230.440212:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829230.440213:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829230.440215:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829230.440222:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787829230.440225:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829230.440227:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1787829230.440229:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829231.334710:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829231.334716:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829231.334775:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829231.334779:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829231.334782:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829232.358692:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829232.358698:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829232.358747:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829232.358751:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829232.358754:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829233.382740:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829233.382751:0:815:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1787829233.382755:0:815:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1787829233.382759:0:815:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829233.382765:0:815:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1787829233.382767:0:815:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829233.382769:0:815:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829233.382772:0:815:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829233.383027:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829233.383031:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829233.383034:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829234.406721:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829234.406728:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829234.406827:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829234.406830:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829234.406833:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829235.430687:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829235.430692:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829235.430763:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829235.430767:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829235.430770:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787829235.558702:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829235.558710:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1787829235.558712:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787829235.558715:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787829235.558718:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787829235.558725:0:40798:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 4036 00000100:00080000:0.0:1787829235.558735:0:40798:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787829235.558739:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829235.558741:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787829235.558745:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787829235.558748:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787829235.558751:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:0.0:1787829235.558754:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:0.0:1787829235.558756:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829235.558758:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:0.0:1787829235.558763:0:40798:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787829235.558775:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829235.558778:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787829235.558782:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787829235.558785:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829235.558787:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1787829235.558790:0:40798:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829235.558799:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787829235.558800:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829235.558803:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787829235.558805:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787829235.558808:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829235.558819:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787829235.558826:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d24000 x1874670646809472/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787829235.558834:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787829235.558837:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829235.558849:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787829235.558851:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829235.558854:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1787829235.558856:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787829235.558858:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787829235.558861:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787829235.558865:0:40798:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4036 00000100:00080000:0.0:1787829235.558871:0:40798:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787829235.558874:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829235.558874:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787829235.558879:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:3.0:1787829235.558880:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787829235.558883:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000020:00000001:0.0:1787829235.558884:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:3.0:1787829235.558886:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000020:00000001:0.0:1787829235.558888:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:3.0:1787829235.558889:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787829235.558893:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 02000000:00000001:3.0:1787829235.558895:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787829235.558898:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000020:00000001:0.0:1787829235.558900:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 02000000:00000001:3.0:1787829235.558901:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829235.558907:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00100000:3.0:1787829235.558910:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d24000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646809472:192.168.203.143@tcp:250:kworker.0 00000100:00000001:0.0:1787829235.558911:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:3.0:1787829235.558919:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00080000:0.0:1787829235.558922:0:40798:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 02000000:00000001:3.0:1787829235.558924:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:0.0:1787829235.558929:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829235.558929:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829235.558936:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787829235.558936:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1787829235.558942:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829235.558945:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787829235.558949:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829235.558952:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1787829235.558957:0:40798:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00000001:3.0:1787829235.558957:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787829235.558960:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601336320 : -126234108215296 : ffff8d30d4d24000) 00000008:00000001:0.0:1787829235.558962:0:40798:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829235.558965:0:40798:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829235.558965:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 02000000:00000001:0.0:1787829235.558971:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787829235.558973:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829235.558974:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787829235.558975:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787829235.558977:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829235.558983:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787829235.558988:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d24f00 x1874670646809600/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787829235.558994:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787829235.558996:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:3.0:1787829235.559000:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:0.0:1787829235.559001:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787829235.559005:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829235.559008:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1787829235.559010:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787829235.559011:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787829235.559013:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1787829235.559014:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787829235.559019:0:40798:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4036 00000100:00000001:3.0:1787829235.559019:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829235.559024:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787829235.559026:0:40798:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787829235.559027:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787829235.559030:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829235.559032:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:3.0:1787829235.559033:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787829235.559035:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787829235.559037:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:3.0:1787829235.559038:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4298709760 00000020:00000001:0.0:1787829235.559040:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000020:00000001:0.0:1787829235.559043:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:0.0:1787829235.559044:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829235.559045:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:3.0:1787829235.559048:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 00000100:00080000:0.0:1787829235.559049:0:40798:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787829235.559053:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829235.559055:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:3.0:1787829235.559056:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4298709760 02000000:00000001:0.0:1787829235.559057:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787829235.559062:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829235.559064:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:3.0:1787829235.559064:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 00000008:00100000:0.0:1787829235.559067:0:40798:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00000001:3.0:1787829235.559069:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000008:00000001:0.0:1787829235.559070:0:40798:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829235.559074:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829235.559075:0:40798:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829235.559078:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 02000000:00000001:0.0:1787829235.559081:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00100000:3.0:1787829235.559085:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829235/real 1787829235] req@ffff8d30d4d24000 x1874670646809472/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829260 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:0.0:1787829235.559092:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:3.0:1787829235.559094:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787829235.559097:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787829235.559098:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:3.0:1787829235.559100:0:2390:0:(events.c:78:reply_in_callback()) Process entered 02000000:00000001:0.0:1787829235.559101:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:3.0:1787829235.559104:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 02000000:00000001:0.0:1787829235.559105:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829235.559108:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829235.559111:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829235.559112:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829235.559116:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787829235.559117:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d25a40 x1874670646809728/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787829235.559127:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00100000:3.0:1787829235.559127:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d24000 x1874670646809472/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829260 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787829235.559131:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829235.559135:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787829235.559140:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829235.559146:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1787829235.559149:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787829235.559151:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787829235.559153:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787829235.559155:0:40798:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4036 00000100:00080000:0.0:1787829235.559157:0:40798:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787829235.559158:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829235.559159:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787829235.559160:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787829235.559161:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787829235.559163:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:3.0:1787829235.559165:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1787829235.559166:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:0.0:1787829235.559169:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829235.559170:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787829235.559171:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:3.0:1787829235.559177:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:0.0:1787829235.559178:0:40798:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:3.0:1787829235.559180:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787829235.559182:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829235.559182:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:0.0:1787829235.559185:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:3.0:1787829235.559187:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 02000000:00000001:0.0:1787829235.559188:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829235.559190:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 02000000:00000001:0.0:1787829235.559192:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787829235.559194:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00000001:0.0:1787829235.559195:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1787829235.559198:0:40798:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1787829235.559202:0:40798:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787829235.559202:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787829235.559205:0:40798:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829235.559206:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 02000000:00000001:0.0:1787829235.559209:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787829235.559212:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829235.559212:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d24000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646809472:192.168.203.143@tcp:250:kworker.0 00000100:00000001:0.0:1787829235.559214:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787829235.559217:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787829235.559219:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829235.559220:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829235.559224:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829235.559224:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829235.559228:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 00000100:00100000:0.0:1787829235.559231:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d270c0 x1874670646809856/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:3.0:1787829235.559231:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829235.559234:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829235.559238:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829235.559240:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 00000100:00000001:0.0:1787829235.559241:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787829235.559244:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829235.559245:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 00000100:00000001:0.0:1787829235.559247:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829235.559249:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829235.559253:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829235.559254:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829235.559256:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829235.559262:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787829235.559264:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829235.559266:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1787829235.559268:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787829235.559269:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787829235.559270:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787829235.559271:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829235.559273:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787829235.559274:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829235.559276:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829235.559279:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d24f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646809600:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829235.559281:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1787829235.559283:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1787829235.559284:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829235.559285:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1787829235.559287:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829235.559292:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787829235.559293:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601340160 : -126234108211456 : ffff8d30d4d24f00) 00000100:00000001:3.0:1787829235.559297:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1787829235.559306:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:3.0:1787829235.559310:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787829235.559311:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829235.559312:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829235.559314:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1787829235.559316:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829235.559319:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298708502 00000100:00000001:3.0:1787829235.559321:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298709760 00000100:00000001:3.0:1787829235.559322:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298708502 00000100:00000001:3.0:1787829235.559324:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298709760 00000100:00000001:3.0:1787829235.559326:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829235.559327:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829235.559328:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1787829235.559332:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829235/real 1787829235] req@ffff8d30d4d24f00 x1874670646809600/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829260 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1787829235.559338:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787829235.559338:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829235.559340:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829235.559341:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829235.559342:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829235.559343:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829235.559344:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829235.559346:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d24f00 x1874670646809600/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829260 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829235.559351:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829235.559352:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829235.559354:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829235.559355:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829235.559355:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829235.559356:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829235.559358:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829235.559359:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829235.559361:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829235.559362:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829235.559364:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d24f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646809600:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829235.559367:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787829235.559367:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787829235.559368:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829235.559370:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787829235.559371:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829235.559372:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829235.559375:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d25a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646809728:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829235.559377:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1787829235.559378:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1787829235.559379:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829235.559380:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1787829235.559382:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829235.559384:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787829235.559385:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601343040 : -126234108208576 : ffff8d30d4d25a40) 00000100:00000001:3.0:1787829235.559387:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1787829235.559393:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:3.0:1787829235.559395:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787829235.559397:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829235.559397:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829235.559399:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1787829235.559400:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829235.559402:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298708502 00000100:00000001:3.0:1787829235.559404:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298709760 00000100:00000001:3.0:1787829235.559405:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298708502 00000100:00000001:3.0:1787829235.559407:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298709760 00000100:00000001:3.0:1787829235.559409:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829235.559410:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829235.559412:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1787829235.559416:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829235/real 1787829235] req@ffff8d30d4d25a40 x1874670646809728/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829260 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1787829235.559424:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787829235.559426:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829235.559428:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829235.559430:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829235.559432:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829235.559433:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829235.559435:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829235.559440:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d25a40 x1874670646809728/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829260 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829235.559454:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829235.559476:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829235.559479:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829235.559481:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829235.559483:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829235.559485:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829235.559486:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829235.559489:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829235.559493:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829235.559495:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829235.559499:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d25a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646809728:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829235.559503:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787829235.559505:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787829235.559507:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829235.559509:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787829235.559511:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829235.559513:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829235.559516:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d270c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646809856:192.168.203.143@tcp:8:kworker.0 00000100:00000001:3.0:1787829235.559520:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1787829235.559522:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1787829235.559523:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829235.559525:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1787829235.559528:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829235.559534:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787829235.559535:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601348800 : -126234108202816 : ffff8d30d4d270c0) 00000100:00000001:3.0:1787829235.559538:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1787829235.559554:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:3.0:1787829235.559558:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787829235.559560:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829235.559561:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829235.559562:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1787829235.559564:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829235.559566:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298708515 00000100:00000001:3.0:1787829235.559568:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298709760 00000100:00000001:3.0:1787829235.559569:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298708515 00000100:00000001:3.0:1787829235.559571:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298709760 00000100:00000001:3.0:1787829235.559572:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829235.559573:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829235.559575:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1787829235.559577:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829235/real 1787829235] req@ffff8d30d4d270c0 x1874670646809856/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829260 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1787829235.559582:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787829235.559583:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829235.559584:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829235.559585:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829235.559586:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829235.559587:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829235.559589:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829235.559591:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d270c0 x1874670646809856/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829260 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829235.559596:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829235.559597:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829235.559598:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829235.559621:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829235.559622:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829235.559624:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829235.559625:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829235.559626:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829235.559629:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829235.559630:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829235.559632:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d270c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646809856:192.168.203.143@tcp:8:kworker.0 00000100:00000001:3.0:1787829235.559635:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829235.559636:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829235.559637:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829235.559638:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829235.559640:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829235.559641:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829235.559642:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829235.559644:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829235.559645:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829235.559647:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829235.559647:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829235.559648:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829235.559649:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829235.559650:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829235.559651:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829235.559652:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829235.559652:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829235.559653:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829235.559654:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829235.559655:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829235.559656:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829235.559657:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829235.559657:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829235.559658:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829235.559659:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829235.559659:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829235.559660:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829235.559661:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829235.559662:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829235.559664:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829235.559664:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829235.559666:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829235.559668:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787829235.559669:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829235.559670:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1787829235.559671:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829236.454672:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829236.454679:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829236.454735:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829236.454739:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829236.454743:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829237.478698:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829237.478706:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829237.478794:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829237.478797:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829237.478800:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829238.502725:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829238.502732:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829238.502798:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829238.502803:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829238.502806:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829239.526704:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829239.526710:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829239.526761:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829239.526764:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829239.526767:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829240.550667:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829240.550673:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829240.550748:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829240.550751:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829240.550754:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787829240.678722:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829240.678730:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1787829240.678732:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787829240.678735:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787829240.678739:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787829240.678745:0:40798:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 4041 00000100:00080000:2.0:1787829240.678753:0:40798:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787829240.678755:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829240.678757:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787829240.678760:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787829240.678763:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787829240.678766:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:2.0:1787829240.678768:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:2.0:1787829240.678770:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829240.678772:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:2.0:1787829240.678775:0:40798:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787829240.678778:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829240.678781:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787829240.678785:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787829240.678788:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829240.678791:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:2.0:1787829240.678794:0:40798:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829240.678805:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787829240.678807:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829240.678811:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787829240.678814:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787829240.678820:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829240.678831:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787829240.678840:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2f0c0 x1874670646809984/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787829240.678847:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787829240.678850:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829240.678861:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787829240.678864:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829240.678867:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1787829240.678868:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787829240.678869:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787829240.678871:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787829240.678873:0:40798:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4041 00000100:00080000:2.0:1787829240.678876:0:40798:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787829240.678877:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829240.678879:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787829240.678881:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787829240.678882:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787829240.678884:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:3.0:1787829240.678884:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:2.0:1787829240.678888:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:3.0:1787829240.678889:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1787829240.678890:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829240.678894:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:3.0:1787829240.678894:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787829240.678898:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00080000:2.0:1787829240.678899:0:40798:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:3.0:1787829240.678901:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829240.678903:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829240.678906:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787829240.678907:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787829240.678910:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787829240.678912:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829240.678913:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829240.678919:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829240.678922:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00100000:3.0:1787829240.678922:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646809984:192.168.203.143@tcp:250:kworker.0 00000008:00100000:2.0:1787829240.678927:0:40798:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00000001:3.0:1787829240.678928:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000008:00000001:2.0:1787829240.678933:0:40798:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829240.678933:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1787829240.678936:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829240.678938:0:40798:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829240.678939:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1787829240.678946:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787829240.678946:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829240.678950:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829240.678955:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787829240.678957:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787829240.678959:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829240.678962:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787829240.678964:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417880768 : -126234291670848 : ffff8d30c9e2f0c0) 00000100:00000001:2.0:1787829240.678966:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829240.678968:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00100000:2.0:1787829240.678982:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2ed00 x1874670646810112/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787829240.678991:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787829240.678994:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829240.678997:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787829240.679001:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:3.0:1787829240.679002:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00080000:2.0:1787829240.679008:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1787829240.679011:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787829240.679013:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787829240.679016:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:3.0:1787829240.679016:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00080000:2.0:1787829240.679022:0:40798:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4041 00000100:00000001:3.0:1787829240.679024:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00080000:2.0:1787829240.679027:0:40798:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787829240.679028:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829240.679030:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829240.679033:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:3.0:1787829240.679033:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000020:00000001:2.0:1787829240.679038:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787829240.679039:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787829240.679041:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000020:00000001:2.0:1787829240.679043:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:3.0:1787829240.679043:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829240.679048:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829240.679049:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:3.0:1787829240.679051:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4298714880 00000100:00080000:2.0:1787829240.679058:0:40798:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787829240.679061:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829240.679063:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:3.0:1787829240.679063:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 02000000:00000001:2.0:1787829240.679068:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787829240.679071:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829240.679071:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4298714880 00000100:00000001:2.0:1787829240.679073:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1787829240.679077:0:40798:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00000001:3.0:1787829240.679080:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 00000008:00000001:2.0:1787829240.679081:0:40798:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829240.679085:0:40798:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829240.679087:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787829240.679090:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:3.0:1787829240.679092:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829240.679093:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829240.679097:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:3.0:1787829240.679098:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 02000000:00000001:2.0:1787829240.679099:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787829240.679103:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829240.679107:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829240.679109:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829240/real 1787829240] req@ffff8d30c9e2f0c0 x1874670646809984/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829265 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829240.679114:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2e940 x1874670646810240/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787829240.679131:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000400:00000001:3.0:1787829240.679131:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000001:2.0:1787829240.679133:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:3.0:1787829240.679135:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1787829240.679137:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787829240.679140:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1787829240.679140:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00080000:2.0:1787829240.679145:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1787829240.679145:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787829240.679147:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787829240.679150:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787829240.679150:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829240.679152:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787829240.679156:0:40798:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4041 00000100:00000001:3.0:1787829240.679156:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829240.679161:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:2.0:1787829240.679162:0:40798:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787829240.679165:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829240.679167:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787829240.679169:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787829240.679171:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00100000:3.0:1787829240.679173:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2f0c0 x1874670646809984/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829265 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:2.0:1787829240.679174:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000020:00000001:2.0:1787829240.679178:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:2.0:1787829240.679182:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829240.679184:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:2.0:1787829240.679191:0:40798:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787829240.679196:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829240.679198:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:2.0:1787829240.679199:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787829240.679202:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829240.679203:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 02000000:00000001:2.0:1787829240.679205:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829240.679209:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:3.0:1787829240.679209:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000008:00100000:2.0:1787829240.679213:0:40798:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00000001:3.0:1787829240.679213:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829240.679215:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829240.679216:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829240.679218:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000008:00000001:2.0:1787829240.679224:0:40798:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787829240.679225:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00000001:2.0:1787829240.679229:0:40798:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787829240.679232:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 02000000:00000001:2.0:1787829240.679233:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:3.0:1787829240.679235:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 02000000:00000001:2.0:1787829240.679236:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829240.679241:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00100000:3.0:1787829240.679243:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646809984:192.168.203.143@tcp:250:kworker.0 02000000:00000001:2.0:1787829240.679244:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787829240.679247:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829240.679251:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829240.679254:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829240.679255:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829240.679257:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829240.679261:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829240.679264:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00100000:2.0:1787829240.679265:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2c000 x1874670646810368/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000020:00000001:3.0:1787829240.679266:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829240.679269:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829240.679274:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829240.679276:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829240.679280:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787829240.679281:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787829240.679282:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829240.679285:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829240.679286:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829240.679292:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1787829240.679293:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829240.679295:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829240.679298:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1787829240.679300:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787829240.679301:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787829240.679303:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787829240.679304:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829240.679306:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787829240.679307:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829240.679308:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829240.679311:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646810112:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829240.679314:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1787829240.679316:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1787829240.679317:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829240.679318:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1787829240.679320:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829240.679325:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787829240.679326:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417879808 : -126234291671808 : ffff8d30c9e2ed00) 00000100:00000001:3.0:1787829240.679330:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1787829240.679340:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:3.0:1787829240.679344:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787829240.679346:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829240.679347:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829240.679348:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1787829240.679350:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829240.679353:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298713622 00000100:00000001:3.0:1787829240.679355:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298714880 00000100:00000001:3.0:1787829240.679357:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298713622 00000100:00000001:3.0:1787829240.679359:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298714880 00000100:00000001:3.0:1787829240.679360:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829240.679362:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829240.679363:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1787829240.679366:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829240/real 1787829240] req@ffff8d30c9e2ed00 x1874670646810112/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829265 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1787829240.679372:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787829240.679373:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829240.679374:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829240.679375:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829240.679376:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829240.679377:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829240.679378:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829240.679380:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2ed00 x1874670646810112/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829265 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829240.679385:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829240.679387:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829240.679388:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829240.679389:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829240.679390:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829240.679391:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829240.679392:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829240.679394:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829240.679396:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829240.679397:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829240.679399:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646810112:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829240.679401:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787829240.679402:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787829240.679403:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829240.679404:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787829240.679406:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829240.679407:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829240.679409:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646810240:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829240.679411:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1787829240.679412:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1787829240.679413:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829240.679414:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1787829240.679416:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829240.679419:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787829240.679420:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417878848 : -126234291672768 : ffff8d30c9e2e940) 00000100:00000001:3.0:1787829240.679423:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1787829240.679448:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:3.0:1787829240.679451:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787829240.679453:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829240.679454:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829240.679455:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1787829240.679457:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829240.679459:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298713622 00000100:00000001:3.0:1787829240.679461:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298714880 00000100:00000001:3.0:1787829240.679463:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298713622 00000100:00000001:3.0:1787829240.679464:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298714880 00000100:00000001:3.0:1787829240.679466:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829240.679467:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829240.679468:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1787829240.679471:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829240/real 1787829240] req@ffff8d30c9e2e940 x1874670646810240/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829265 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1787829240.679476:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787829240.679476:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829240.679478:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829240.679479:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829240.679480:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829240.679481:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829240.679482:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829240.679484:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2e940 x1874670646810240/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829265 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829240.679493:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829240.679495:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829240.679496:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829240.679497:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829240.679498:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829240.679499:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829240.679500:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829240.679501:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829240.679503:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829240.679505:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829240.679506:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646810240:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829240.679509:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787829240.679509:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787829240.679510:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829240.679512:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787829240.679513:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829240.679514:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829240.679516:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646810368:192.168.203.143@tcp:8:kworker.0 00000100:00000001:3.0:1787829240.679518:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1787829240.679519:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1787829240.679521:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829240.679522:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1787829240.679523:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829240.679526:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787829240.679527:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417868288 : -126234291683328 : ffff8d30c9e2c000) 00000100:00000001:3.0:1787829240.679529:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1787829240.679535:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:3.0:1787829240.679538:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787829240.679539:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829240.679540:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829240.679541:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1787829240.679543:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829240.679545:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298713635 00000100:00000001:3.0:1787829240.679547:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298714880 00000100:00000001:3.0:1787829240.679548:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298713635 00000100:00000001:3.0:1787829240.679550:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298714880 00000100:00000001:3.0:1787829240.679551:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829240.679552:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829240.679553:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1787829240.679555:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829240/real 1787829240] req@ffff8d30c9e2c000 x1874670646810368/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829265 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1787829240.679560:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787829240.679561:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829240.679562:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829240.679563:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829240.679564:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829240.679565:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829240.679565:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829240.679568:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2c000 x1874670646810368/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829265 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829240.679572:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829240.679573:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829240.679575:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829240.679576:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829240.679577:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829240.679578:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829240.679579:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829240.679580:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829240.679582:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829240.679583:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829240.679585:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646810368:192.168.203.143@tcp:8:kworker.0 00000100:00000001:3.0:1787829240.679588:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829240.679589:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829240.679590:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829240.679591:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829240.679592:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829240.679593:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829240.679594:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829240.679596:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829240.679597:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829240.679627:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829240.679629:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829240.679630:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829240.679631:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829240.679632:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829240.679633:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829240.679634:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829240.679635:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829240.679636:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829240.679637:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829240.679638:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829240.679639:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829240.679640:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829240.679640:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829240.679641:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829240.679642:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829240.679643:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829240.679643:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829240.679644:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829240.679645:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829240.679646:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829240.679647:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829240.679648:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829240.679650:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787829240.679651:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829240.679652:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1787829240.679653:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829241.574681:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829241.574686:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829241.574729:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829241.574734:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829241.574738:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829242.598718:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829242.598725:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829242.598776:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829242.598779:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829242.598782:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829243.622749:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829243.622773:0:815:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1787829243.622776:0:815:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1787829243.622780:0:815:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829243.622785:0:815:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1787829243.622787:0:815:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829243.622789:0:815:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829243.622792:0:815:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829243.622902:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829243.622905:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829243.622908:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829244.646722:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829244.646729:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829244.646779:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829244.646784:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829244.646788:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829245.670673:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829245.670681:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829245.670726:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829245.670729:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829245.670732:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787829245.798736:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829245.798745:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1787829245.798747:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787829245.798750:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787829245.798754:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787829245.798761:0:40798:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 4046 00000100:00080000:0.0:1787829245.798769:0:40798:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787829245.798783:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829245.798785:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787829245.798790:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787829245.798792:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787829245.798796:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:0.0:1787829245.798799:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:0.0:1787829245.798801:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829245.798802:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:0.0:1787829245.798808:0:40798:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787829245.798811:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829245.798814:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787829245.798818:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787829245.798821:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829245.798824:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1787829245.798827:0:40798:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829245.798850:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787829245.798852:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829245.798855:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787829245.798857:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787829245.798861:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829245.798870:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787829245.798878:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d27c00 x1874670646810496/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787829245.798887:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787829245.798890:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829245.798913:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787829245.798915:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829245.798918:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1787829245.798919:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787829245.798920:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787829245.798922:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787829245.798924:0:40798:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4046 00000100:00080000:0.0:1787829245.798927:0:40798:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787829245.798928:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829245.798929:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787829245.798931:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787829245.798932:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787829245.798934:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000020:00000001:0.0:1787829245.798935:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:0.0:1787829245.798936:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829245.798937:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:0.0:1787829245.798939:0:40798:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787829245.798941:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829245.798942:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787829245.798946:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787829245.798947:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829245.798949:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1787829245.798952:0:40798:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1787829245.798954:0:40798:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829245.798956:0:40798:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829245.798959:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787829245.798960:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829245.798961:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787829245.798962:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787829245.798964:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829245.798967:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829245.798967:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1787829245.798984:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00100000:0.0:1787829245.798985:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d252c0 x1874670646810624/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787829245.798987:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787829245.798991:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787829245.798993:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829245.798994:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787829245.798998:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829245.798999:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:0.0:1787829245.799001:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829245.799002:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00080000:0.0:1787829245.799004:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 02000000:00000001:3.0:1787829245.799007:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787829245.799008:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1787829245.799011:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787829245.799013:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1787829245.799014:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d27c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646810496:192.168.203.143@tcp:250:kworker.0 00000100:00000001:0.0:1787829245.799016:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:3.0:1787829245.799020:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00080000:0.0:1787829245.799022:0:40798:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4046 02000000:00000001:3.0:1787829245.799025:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000100:00080000:0.0:1787829245.799026:0:40798:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787829245.799029:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 02000000:00000001:3.0:1787829245.799029:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829245.799030:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 02000000:00000001:3.0:1787829245.799033:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000020:00000001:0.0:1787829245.799034:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787829245.799036:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787829245.799037:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 02000000:00000001:3.0:1787829245.799039:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787829245.799040:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:0.0:1787829245.799044:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829245.799045:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:0.0:1787829245.799047:0:40798:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787829245.799049:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829245.799050:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787829245.799052:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787829245.799053:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829245.799054:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1787829245.799057:0:40798:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00000001:3.0:1787829245.799057:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000008:00000001:0.0:1787829245.799061:0:40798:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829245.799062:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601351680 : -126234108199936 : ffff8d30d4d27c00) 00000100:00000001:0.0:1787829245.799063:0:40798:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829245.799068:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:3.0:1787829245.799070:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 02000000:00000001:0.0:1787829245.799071:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829245.799074:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787829245.799075:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787829245.799076:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829245.799080:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787829245.799083:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d25680 x1874670646810752/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787829245.799089:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787829245.799090:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829245.799092:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787829245.799094:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829245.799095:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1787829245.799097:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787829245.799098:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787829245.799099:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787829245.799101:0:40798:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4046 00000100:00080000:0.0:1787829245.799104:0:40798:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787829245.799105:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829245.799106:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787829245.799107:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787829245.799107:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787829245.799109:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000020:00000001:0.0:1787829245.799110:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:0.0:1787829245.799114:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000800:00000100:3.0:1787829245.799115:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:0.0:1787829245.799116:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:0.0:1787829245.799121:0:40798:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787829245.799123:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829245.799125:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787829245.799127:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787829245.799129:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829245.799131:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:3.0:1787829245.799131:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000008:00100000:0.0:1787829245.799135:0:40798:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1787829245.799137:0:40798:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829245.799137:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829245.799141:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829245.799142:0:40798:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829245.799147:0:2390:0:(events.c:68:request_out_callback()) Process leaving 02000000:00000001:0.0:1787829245.799150:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787829245.799152:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829245.799153:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829245.799154:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787829245.799157:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787829245.799160:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829245.799160:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4298720000 00000100:00000001:0.0:1787829245.799166:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787829245.799171:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d26940 x1874670646810880/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787829245.799171:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 00000100:00000001:3.0:1787829245.799180:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4298720000 00000100:00000001:3.0:1787829245.799184:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 00000100:00000001:3.0:1787829245.799187:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787829245.799189:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787829245.799190:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829245.799194:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829245.799195:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00000001:0.0:1787829245.799198:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829245.799203:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829245/real 1787829245] req@ffff8d30d4d27c00 x1874670646810496/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829270 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1787829245.799211:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787829245.799213:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829245.799216:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829245.799219:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829245.799221:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829245.799223:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829245.799225:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829245.799228:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d27c00 x1874670646810496/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829270 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829245.799233:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829245.799235:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829245.799237:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829245.799238:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829245.799239:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829245.799252:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829245.799254:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829245.799257:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829245.799261:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829245.799264:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829245.799268:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d27c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646810496:192.168.203.143@tcp:250:kworker.0 00000100:00000001:3.0:1787829245.799273:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829245.799281:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829245.799283:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829245.799285:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829245.799289:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829245.799291:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829245.799292:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829245.799296:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829245.799298:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829245.799302:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829245.799304:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829245.799306:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829245.799313:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787829245.799316:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829245.799318:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1787829245.799320:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787829245.799321:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787829245.799323:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787829245.799324:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829245.799327:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787829245.799328:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829245.799331:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829245.799335:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d252c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646810624:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829245.799339:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1787829245.799341:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1787829245.799343:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829245.799345:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1787829245.799348:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829245.799354:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787829245.799356:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601341120 : -126234108210496 : ffff8d30d4d252c0) 00000100:00000001:3.0:1787829245.799361:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1787829245.799373:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:3.0:1787829245.799378:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787829245.799381:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829245.799382:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829245.799385:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1787829245.799387:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829245.799406:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298718742 00000100:00000001:3.0:1787829245.799409:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298720000 00000100:00000001:3.0:1787829245.799412:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298718742 00000100:00000001:3.0:1787829245.799415:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298720000 00000100:00000001:3.0:1787829245.799417:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829245.799419:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829245.799421:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1787829245.799426:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829245/real 1787829245] req@ffff8d30d4d252c0 x1874670646810624/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829270 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1787829245.799437:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787829245.799438:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829245.799441:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829245.799443:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829245.799445:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829245.799447:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829245.799448:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829245.799453:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d252c0 x1874670646810624/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829270 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829245.799461:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829245.799464:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829245.799467:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829245.799469:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829245.799470:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829245.799472:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829245.799474:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829245.799477:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829245.799481:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829245.799483:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829245.799487:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d252c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646810624:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829245.799492:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787829245.799493:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787829245.799495:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829245.799497:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787829245.799499:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829245.799501:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829245.799504:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d25680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646810752:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829245.799508:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1787829245.799510:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1787829245.799512:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829245.799514:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1787829245.799517:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829245.799533:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787829245.799535:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601342080 : -126234108209536 : ffff8d30d4d25680) 00000100:00000001:3.0:1787829245.799539:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1787829245.799551:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:3.0:1787829245.799557:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787829245.799559:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829245.799561:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829245.799563:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1787829245.799565:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829245.799569:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298718742 00000100:00000001:3.0:1787829245.799572:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298720000 00000100:00000001:3.0:1787829245.799574:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298718742 00000100:00000001:3.0:1787829245.799577:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298720000 00000100:00000001:3.0:1787829245.799580:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829245.799582:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829245.799584:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1787829245.799588:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829245/real 1787829245] req@ffff8d30d4d25680 x1874670646810752/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829270 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1787829245.799597:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787829245.799637:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829245.799641:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829245.799643:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829245.799644:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829245.799646:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829245.799648:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829245.799652:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d25680 x1874670646810752/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829270 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829245.799661:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829245.799663:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829245.799667:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829245.799668:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829245.799670:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829245.799671:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829245.799673:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829245.799676:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829245.799680:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829245.799682:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829245.799686:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d25680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646810752:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829245.799691:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787829245.799692:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787829245.799694:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829245.799696:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787829245.799698:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829245.799701:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829245.799704:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d26940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646810880:192.168.203.143@tcp:8:kworker.0 00000100:00000001:3.0:1787829245.799708:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1787829245.799710:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1787829245.799712:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829245.799713:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1787829245.799716:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829245.799722:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787829245.799724:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601346880 : -126234108204736 : ffff8d30d4d26940) 00000100:00000001:3.0:1787829245.799728:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1787829245.799741:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:3.0:1787829245.799746:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787829245.799748:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829245.799750:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829245.799752:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1787829245.799755:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829245.799759:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298718756 00000100:00000001:3.0:1787829245.799762:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298720000 00000100:00000001:3.0:1787829245.799764:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298718756 00000100:00000001:3.0:1787829245.799767:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298720000 00000100:00000001:3.0:1787829245.799769:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829245.799771:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829245.799784:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1787829245.799788:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829245/real 1787829245] req@ffff8d30d4d26940 x1874670646810880/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829270 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1787829245.799797:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787829245.799798:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829245.799800:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829245.799801:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829245.799803:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829245.799804:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829245.799805:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829245.799807:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d26940 x1874670646810880/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829270 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829245.799812:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829245.799814:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829245.799815:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829245.799816:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829245.799817:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829245.799818:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829245.799819:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829245.799820:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829245.799822:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829245.799823:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829245.799825:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d26940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646810880:192.168.203.143@tcp:8:kworker.0 00000100:00000001:3.0:1787829245.799828:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829245.799830:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829245.799831:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829245.799832:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829245.799834:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829245.799835:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829245.799836:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829245.799837:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829245.799838:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829245.799840:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829245.799841:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829245.799842:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829245.799843:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829245.799844:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829245.799845:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829245.799846:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829245.799846:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829245.799847:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829245.799848:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829245.799849:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829245.799850:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829245.799851:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829245.799852:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829245.799853:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829245.799853:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829245.799854:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829245.799855:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829245.799856:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829245.799857:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829245.799858:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829245.799859:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829245.799860:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829245.799862:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787829245.799863:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829245.799864:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1787829245.799866:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829246.694743:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829246.694751:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829246.694847:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829246.694853:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829246.694858:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829247.718699:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829247.718716:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829247.718888:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829247.718895:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829247.718900:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829248.742685:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829248.742690:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829248.742745:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829248.742748:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829248.742751:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829249.766768:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829249.766777:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829249.766870:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829249.766874:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829249.766878:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829250.790693:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829250.790698:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829250.790754:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829250.790773:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829250.790776:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787829250.918725:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829250.918736:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1787829250.918738:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1787829250.918743:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787829250.918767:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1787829250.918773:0:468:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 4051 00000100:00080000:1.0:1787829250.918785:0:468:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1787829250.918788:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787829250.918791:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:1.0:1787829250.918796:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:1.0:1787829250.918799:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:1.0:1787829250.918802:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:1.0:1787829250.918805:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:1.0:1787829250.918808:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787829250.918809:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:1.0:1787829250.918814:0:468:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:1.0:1787829250.918828:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829250.918831:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1787829250.918834:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829250.918838:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829250.918840:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:1.0:1787829250.918843:0:468:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829250.918853:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1787829250.918854:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829250.918856:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1787829250.918858:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1787829250.918861:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829250.918877:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829250.918885:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d13ecb40 x1874670646811008/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787829250.918896:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787829250.918901:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829250.918916:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1787829250.918920:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829250.918923:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1787829250.918932:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1787829250.918936:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787829250.918940:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1787829250.918944:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4051 00000100:00080000:1.0:1787829250.918950:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787829250.918950:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787829250.918955:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829250.918955:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1787829250.918959:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:3.0:1787829250.918960:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787829250.918963:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787829250.918965:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829250.918966:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:1.0:1787829250.918970:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 02000000:00000001:3.0:1787829250.918970:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000020:00000001:1.0:1787829250.918973:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 02000000:00000001:3.0:1787829250.918974:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829250.918977:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829250.918980:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00100000:3.0:1787829250.918983:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d13ecb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646811008:192.168.203.143@tcp:250:kworker.0 00000100:00000001:1.0:1787829250.918986:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829250.918989:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:1.0:1787829250.918990:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 02000000:00000001:3.0:1787829250.918993:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1787829250.918996:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829250.918999:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00080000:1.0:1787829250.919000:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 02000000:00000001:3.0:1787829250.919004:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829250.919006:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829250.919010:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1787829250.919016:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829250.919019:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 02000000:00000001:1.0:1787829250.919020:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829250.919023:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839541340992 : -126234168210624 : ffff8d30d13ecb40) 00000100:00000001:1.0:1787829250.919025:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:1.0:1787829250.919031:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00000001:3.0:1787829250.919031:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000008:00000001:1.0:1787829250.919037:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829250.919040:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829250.919046:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1787829250.919047:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829250.919049:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1787829250.919051:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1787829250.919053:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829250.919059:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829250.919066:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d13ec000 x1874670646811136/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787829250.919076:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787829250.919078:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:3.0:1787829250.919094:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787829250.919096:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1787829250.919101:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829250.919106:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1787829250.919107:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787829250.919109:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:3.0:1787829250.919112:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00080000:1.0:1787829250.919113:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787829250.919116:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829250.919118:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:3.0:1787829250.919119:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1787829250.919126:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1787829250.919127:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4051 00000100:00000001:3.0:1787829250.919130:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4298725120 00000100:00080000:1.0:1787829250.919134:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1787829250.919138:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829250.919140:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 00000100:00000001:1.0:1787829250.919141:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:1.0:1787829250.919146:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:1.0:1787829250.919147:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:1.0:1787829250.919150:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000020:00000001:1.0:1787829250.919152:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:1.0:1787829250.919155:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829250.919155:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4298725120 00000100:00000001:1.0:1787829250.919159:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:3.0:1787829250.919165:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 00000100:00080000:1.0:1787829250.919166:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:3.0:1787829250.919170:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829250.919172:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829250.919173:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829250.919177:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:3.0:1787829250.919178:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 02000000:00000001:1.0:1787829250.919183:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00100000:3.0:1787829250.919185:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829250/real 1787829250] req@ffff8d30d13ecb40 x1874670646811008/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829275 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:1.0:1787829250.919200:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829250.919206:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:1.0:1787829250.919210:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000400:00000001:3.0:1787829250.919212:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787829250.919215:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000008:00000001:1.0:1787829250.919216:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829250.919218:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787829250.919221:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829250.919222:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829250.919225:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829250.919227:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 02000000:00000001:1.0:1787829250.919228:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:3.0:1787829250.919229:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:1.0:1787829250.919233:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829250.919234:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d13ecb40 x1874670646811008/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829275 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787829250.919237:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1787829250.919241:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:3.0:1787829250.919245:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:1.0:1787829250.919246:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829250.919248:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829250.919263:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829250.919264:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829250.919265:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829250.919266:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829250.919267:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829250.919270:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829250.919273:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:1.0:1787829250.919278:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d13ec780 x1874670646811264/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829250.919278:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829250.919282:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829250.919287:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d13ecb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646811008:192.168.203.143@tcp:250:kworker.0 00000100:00000001:3.0:1787829250.919295:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829250.919298:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829250.919300:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829250.919302:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829250.919306:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829250.919309:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829250.919312:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 00000100:00000001:1.0:1787829250.919313:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 02000000:00000001:3.0:1787829250.919316:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 00000100:00000001:1.0:1787829250.919317:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829250.919318:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829250.919322:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787829250.919324:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829250.919324:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829250.919329:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:1.0:1787829250.919332:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1787829250.919335:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00080000:1.0:1787829250.919337:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1787829250.919338:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829250.919343:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787829250.919344:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:3.0:1787829250.919345:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787829250.919348:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00080000:1.0:1787829250.919349:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787829250.919349:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787829250.919366:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829250.919370:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787829250.919371:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829250.919376:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829250.919377:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00100000:3.0:1787829250.919379:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d13ec000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646811136:192.168.203.143@tcp:38:kworker.0 00000100:00080000:1.0:1787829250.919385:0:468:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4051 00000100:00000001:3.0:1787829250.919385:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1787829250.919388:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1787829250.919392:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1787829250.919394:0:468:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 02000000:00000001:3.0:1787829250.919394:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1787829250.919396:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829250.919400:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829250.919403:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787829250.919404:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:3.0:1787829250.919406:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839541338112 : -126234168213504 : ffff8d30d13ec000) 00000020:00000001:1.0:1787829250.919411:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:3.0:1787829250.919413:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000020:00000001:1.0:1787829250.919416:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:1.0:1787829250.919420:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000020:00000001:1.0:1787829250.919423:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000800:00000100:3.0:1787829250.919428:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787829250.919429:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787829250.919433:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:3.0:1787829250.919433:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787829250.919436:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829250.919437:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829250.919438:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1787829250.919440:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829250.919443:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298723862 00000100:00000001:3.0:1787829250.919447:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298725120 00000100:00080000:1.0:1787829250.919451:0:468:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:3.0:1787829250.919451:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298723862 00000100:00000001:3.0:1787829250.919455:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298725120 00000100:00000001:1.0:1787829250.919459:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829250.919459:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829250.919464:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829250.919466:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:3.0:1787829250.919467:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 02000000:00000001:1.0:1787829250.919472:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00100000:3.0:1787829250.919473:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829250/real 1787829250] req@ffff8d30d13ec000 x1874670646811136/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829275 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:1.0:1787829250.919480:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:3.0:1787829250.919485:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000001:1.0:1787829250.919487:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000400:00000001:3.0:1787829250.919488:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000008:00100000:1.0:1787829250.919492:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000008:00000001:1.0:1787829250.919497:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829250.919499:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829250.919500:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829250.919503:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 02000000:00000001:1.0:1787829250.919505:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:3.0:1787829250.919505:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829250.919509:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 02000000:00000001:1.0:1787829250.919510:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829250.919511:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829250.919518:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00100000:3.0:1787829250.919518:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d13ec000 x1874670646811136/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829275 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:1.0:1787829250.919522:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1787829250.919527:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829250.919528:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829250.919533:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829250.919535:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829250.919537:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829250.919537:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829250.919539:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829250.919543:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829250.919545:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829250.919548:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:1.0:1787829250.919550:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d13ec3c0 x1874670646811392/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829250.919552:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829250.919555:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829250.919561:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d13ec000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646811136:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829250.919568:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787829250.919570:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787829250.919572:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829250.919577:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 02000000:00000001:3.0:1787829250.919577:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787829250.919578:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829250.919583:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829250.919583:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829250.919589:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d13ec780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646811264:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787829250.919592:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829250.919592:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1787829250.919595:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1787829250.919598:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829250.919599:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1787829250.919621:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829250.919625:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787829250.919626:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839541340032 : -126234168211584 : ffff8d30d13ec780) 00000100:00000001:3.0:1787829250.919629:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1787829250.919638:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:3.0:1787829250.919642:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787829250.919644:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829250.919646:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829250.919647:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1787829250.919650:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829250.919653:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298723863 00000100:00000001:3.0:1787829250.919655:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298725120 00000100:00000001:3.0:1787829250.919657:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298723863 00000100:00000001:3.0:1787829250.919659:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298725120 00000100:00000001:3.0:1787829250.919661:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829250.919662:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829250.919663:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1787829250.919666:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829250/real 1787829250] req@ffff8d30d13ec780 x1874670646811264/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829275 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1787829250.919671:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787829250.919671:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829250.919673:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829250.919674:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829250.919675:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829250.919676:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829250.919677:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829250.919679:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d13ec780 x1874670646811264/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829275 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829250.919684:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829250.919685:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829250.919686:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829250.919687:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829250.919688:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829250.919689:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829250.919690:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829250.919691:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829250.919694:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829250.919696:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829250.919698:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d13ec780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646811264:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829250.919701:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829250.919702:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829250.919703:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829250.919704:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829250.919705:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829250.919706:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829250.919707:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829250.919709:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829250.919709:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829250.919710:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829250.919711:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829250.919712:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829250.919713:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829250.919714:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829250.919715:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829250.919715:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829250.919716:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829250.919717:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829250.919718:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829250.919719:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829250.919720:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829250.919721:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829250.919723:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787829250.919724:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829250.919725:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1787829250.919726:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787829250.919727:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787829250.919728:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787829250.919729:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829250.919730:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787829250.919731:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829250.919732:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829250.919734:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d13ec3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646811392:192.168.203.143@tcp:8:kworker.0 00000100:00000001:3.0:1787829250.919736:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1787829250.919738:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1787829250.919739:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829250.919740:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1787829250.919742:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829250.919745:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787829250.919753:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839541339072 : -126234168212544 : ffff8d30d13ec3c0) 00000100:00000001:3.0:1787829250.919756:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1787829250.919763:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:3.0:1787829250.919766:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787829250.919768:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829250.919768:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829250.919770:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1787829250.919771:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829250.919773:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298723876 00000100:00000001:3.0:1787829250.919775:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298725120 00000100:00000001:3.0:1787829250.919776:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298723876 00000100:00000001:3.0:1787829250.919778:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298725120 00000100:00000001:3.0:1787829250.919779:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829250.919780:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829250.919782:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1787829250.919784:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829250/real 1787829250] req@ffff8d30d13ec3c0 x1874670646811392/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829275 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1787829250.919789:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787829250.919789:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829250.919798:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829250.919800:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829250.919800:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829250.919802:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829250.919802:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829250.919805:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d13ec3c0 x1874670646811392/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829275 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829250.919809:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829250.919811:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829250.919813:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829250.919814:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829250.919815:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829250.919816:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829250.919817:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829250.919819:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829250.919821:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829250.919822:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829250.919824:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d13ec3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646811392:192.168.203.143@tcp:8:kworker.0 00000100:00000001:3.0:1787829250.919826:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829250.919827:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829250.919828:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829250.919829:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829250.919830:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829250.919830:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829250.919831:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829250.919832:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829250.919833:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829250.919834:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829250.919835:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829250.919836:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829250.919838:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787829250.919839:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829250.919840:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1787829250.919841:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829251.814712:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829251.814719:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829251.814779:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829251.814783:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829251.814786:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829252.838722:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829252.838729:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829252.838825:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829252.838830:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829252.838834:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829253.862713:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829253.862720:0:815:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1787829253.862723:0:815:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1787829253.862725:0:815:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829253.862730:0:815:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1787829253.862732:0:815:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829253.862734:0:815:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829253.862736:0:815:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829253.862770:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829253.862774:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829253.862776:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829254.886714:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829254.886719:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829254.886776:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829254.886780:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829254.886782:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829255.910745:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829255.910755:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829255.910889:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829255.910898:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829255.910905:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787829256.038699:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829256.038708:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1787829256.038710:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787829256.038722:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787829256.038726:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787829256.038732:0:468:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 4056 00000100:00080000:2.0:1787829256.038743:0:468:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787829256.038745:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829256.038747:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787829256.038751:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787829256.038753:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787829256.038756:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:2.0:1787829256.038758:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:2.0:1787829256.038760:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829256.038762:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:2.0:1787829256.038768:0:468:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787829256.038771:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829256.038775:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787829256.038779:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787829256.038783:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829256.038785:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:2.0:1787829256.038788:0:468:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829256.038801:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787829256.038804:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829256.038808:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787829256.038811:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787829256.038817:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829256.038831:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787829256.038842:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2da40 x1874670646811520/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787829256.038853:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787829256.038856:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829256.038872:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787829256.038875:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829256.038878:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1787829256.038880:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787829256.038881:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787829256.038884:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787829256.038886:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4056 00000100:00080000:2.0:1787829256.038890:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787829256.038892:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829256.038893:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787829256.038895:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787829256.038896:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787829256.038899:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000020:00000001:2.0:1787829256.038901:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:2.0:1787829256.038903:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829256.038904:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:2.0:1787829256.038908:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787829256.038910:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829256.038912:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787829256.038917:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787829256.038919:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829256.038922:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:3.0:1787829256.038922:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000008:00100000:2.0:1787829256.038930:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00000001:3.0:1787829256.038930:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787829256.038935:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000008:00000001:2.0:1787829256.038936:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829256.038940:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829256.038941:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787829256.038945:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829256.038946:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787829256.038950:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829256.038951:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:2.0:1787829256.038955:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787829256.038957:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787829256.038958:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787829256.038963:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829256.038964:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829256.038970:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829256.038972:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2da40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646811520:192.168.203.143@tcp:250:kworker.0 00000100:00100000:2.0:1787829256.038980:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2f480 x1874670646811648/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787829256.038982:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1787829256.038988:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:2.0:1787829256.038993:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 02000000:00000001:3.0:1787829256.038993:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829256.038997:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829256.039009:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829256.039010:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00080000:2.0:1787829256.039012:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829256.039016:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1787829256.039017:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 02000000:00000001:3.0:1787829256.039018:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787829256.039019:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787829256.039026:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787829256.039029:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4056 00000100:00080000:2.0:1787829256.039032:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787829256.039034:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829256.039035:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787829256.039037:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:3.0:1787829256.039039:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000020:00000001:2.0:1787829256.039040:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:3.0:1787829256.039044:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417875008 : -126234291676608 : ffff8d30c9e2da40) 00000020:00000001:2.0:1787829256.039046:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:3.0:1787829256.039050:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000020:00000001:2.0:1787829256.039053:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:2.0:1787829256.039059:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829256.039061:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:2.0:1787829256.039066:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787829256.039070:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829256.039073:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787829256.039078:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787829256.039081:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829256.039084:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1787829256.039088:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000800:00000100:3.0:1787829256.039090:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000008:00000001:2.0:1787829256.039094:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829256.039099:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829256.039105:0:2390:0:(events.c:34:request_out_callback()) Process entered 02000000:00000001:2.0:1787829256.039108:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787829256.039111:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829256.039112:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787829256.039113:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787829256.039117:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:3.0:1787829256.039119:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829256.039120:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829256.039122:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1787829256.039127:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829256.039130:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829256.039136:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4298730240 00000100:00100000:2.0:1787829256.039137:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2c000 x1874670646811776/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787829256.039147:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787829256.039148:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 00000100:00000001:2.0:1787829256.039149:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829256.039156:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787829256.039159:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829256.039170:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1787829256.039171:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787829256.039173:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787829256.039175:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787829256.039177:0:468:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4056 00000100:00000001:3.0:1787829256.039180:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4298730240 00000100:00080000:2.0:1787829256.039181:0:468:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787829256.039187:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829256.039189:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:3.0:1787829256.039191:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 00000020:00000001:2.0:1787829256.039192:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787829256.039195:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787829256.039219:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:3.0:1787829256.039219:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000020:00000001:2.0:1787829256.039228:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:3.0:1787829256.039230:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829256.039242:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00000001:2.0:1787829256.039245:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829256.039249:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00100000:3.0:1787829256.039255:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829256/real 1787829256] req@ffff8d30c9e2da40 x1874670646811520/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829281 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829256.039258:0:468:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787829256.039266:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829256.039272:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787829256.039277:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787829256.039283:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829256.039288:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1787829256.039292:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000008:00000001:2.0:1787829256.039302:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829256.039304:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:3.0:1787829256.039305:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787829256.039309:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 02000000:00000001:2.0:1787829256.039310:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:3.0:1787829256.039315:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829256.039337:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829256.039339:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:2.0:1787829256.039341:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829256.039343:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787829256.039346:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:3.0:1787829256.039348:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:2.0:1787829256.039350:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787829256.039355:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829256.039360:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2da40 x1874670646811520/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829281 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787829256.039364:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787829256.039395:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2e940 x1874670646811904/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787829256.039403:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829256.039410:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787829256.039411:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787829256.039416:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829256.039418:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829256.039420:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829256.039423:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829256.039426:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829256.039429:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829256.039431:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829256.039434:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829256.039439:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829256.039441:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829256.039445:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2da40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646811520:192.168.203.143@tcp:250:kworker.0 00000100:00000001:3.0:1787829256.039457:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829256.039460:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829256.039461:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829256.039464:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829256.039467:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829256.039468:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829256.039471:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829256.039476:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829256.039477:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829256.039499:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829256.039500:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829256.039503:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829256.039511:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787829256.039513:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829256.039516:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1787829256.039518:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787829256.039520:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787829256.039522:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787829256.039524:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829256.039527:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787829256.039529:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829256.039531:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829256.039535:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646811648:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829256.039539:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1787829256.039541:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1787829256.039543:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829256.039545:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1787829256.039549:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829256.039576:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787829256.039577:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417881728 : -126234291669888 : ffff8d30c9e2f480) 00000100:00000001:3.0:1787829256.039583:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1787829256.039604:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:3.0:1787829256.039612:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787829256.039616:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829256.039617:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829256.039619:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1787829256.039623:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829256.039628:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298728983 00000100:00000001:3.0:1787829256.039676:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298730240 00000100:00000001:3.0:1787829256.039681:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298728983 00000100:00000001:3.0:1787829256.039685:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298730240 00000100:00000001:3.0:1787829256.039687:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829256.039690:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829256.039692:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1787829256.039698:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829256/real 1787829256] req@ffff8d30c9e2f480 x1874670646811648/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829281 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1787829256.039715:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787829256.039717:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829256.039732:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829256.039737:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829256.039740:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829256.039743:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829256.039746:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829256.039754:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2f480 x1874670646811648/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829281 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829256.039767:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829256.039772:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829256.039777:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829256.039780:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829256.039888:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829256.039891:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829256.039894:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829256.039897:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829256.040018:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829256.040020:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829256.040024:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646811648:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829256.040029:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787829256.040031:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787829256.040033:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829256.040036:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787829256.040038:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829256.040040:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829256.040043:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646811776:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829256.040046:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1787829256.040047:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1787829256.040050:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829256.040052:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1787829256.040056:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829256.040063:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787829256.040065:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417868288 : -126234291683328 : ffff8d30c9e2c000) 00000100:00000001:3.0:1787829256.040068:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1787829256.040082:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:3.0:1787829256.040132:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787829256.040136:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829256.040138:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829256.040140:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1787829256.040142:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829256.040146:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298728983 00000100:00000001:3.0:1787829256.040148:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298730240 00000100:00000001:3.0:1787829256.040150:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298728983 00000100:00000001:3.0:1787829256.040152:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298730240 00000100:00000001:3.0:1787829256.040154:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829256.040156:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829256.040158:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1787829256.040161:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829256/real 1787829256] req@ffff8d30c9e2c000 x1874670646811776/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829281 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1787829256.040177:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787829256.040178:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829256.040180:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829256.040181:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829256.040183:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829256.040184:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829256.040185:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829256.040188:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2c000 x1874670646811776/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829281 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829256.040194:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829256.040196:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829256.040198:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829256.040199:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829256.040200:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829256.040202:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829256.040203:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829256.040205:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829256.040208:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829256.040209:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829256.040212:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646811776:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829256.040214:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787829256.040215:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787829256.040216:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829256.040218:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787829256.040220:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829256.040221:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829256.040223:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646811904:192.168.203.143@tcp:8:kworker.0 00000100:00000001:3.0:1787829256.040225:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1787829256.040227:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1787829256.040228:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829256.040240:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1787829256.040244:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829256.040248:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787829256.040249:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417878848 : -126234291672768 : ffff8d30c9e2e940) 00000100:00000001:3.0:1787829256.040252:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1787829256.040262:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:3.0:1787829256.040265:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787829256.040267:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829256.040268:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829256.040270:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1787829256.040272:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829256.040274:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298728996 00000100:00000001:3.0:1787829256.040276:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298730240 00000100:00000001:3.0:1787829256.040278:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298728996 00000100:00000001:3.0:1787829256.040280:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298730240 00000100:00000001:3.0:1787829256.040282:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829256.040283:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829256.040285:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1787829256.040288:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829256/real 1787829256] req@ffff8d30c9e2e940 x1874670646811904/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829281 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1787829256.040293:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787829256.040294:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829256.040296:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829256.040297:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829256.040298:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829256.040299:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829256.040300:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829256.040303:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2e940 x1874670646811904/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829281 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829256.040309:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829256.040310:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829256.040312:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829256.040314:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829256.040315:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829256.040316:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829256.040340:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829256.040342:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829256.040345:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829256.040347:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829256.040350:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646811904:192.168.203.143@tcp:8:kworker.0 00000100:00000001:3.0:1787829256.040353:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829256.040355:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829256.040357:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829256.040358:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829256.040360:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829256.040362:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829256.040363:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829256.040375:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829256.040376:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829256.040379:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829256.040380:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829256.040382:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829256.040383:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829256.040384:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829256.040385:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829256.040386:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829256.040387:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829256.040388:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829256.040389:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829256.040391:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829256.040392:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829256.040394:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829256.040395:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829256.040396:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829256.040397:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829256.040398:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829256.040399:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829256.040400:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829256.040401:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829256.040403:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829256.040404:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829256.040406:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829256.040409:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787829256.040410:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829256.040411:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1787829256.040413:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829256.934682:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829256.934703:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829256.934776:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829256.934780:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829256.934783:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829257.958710:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829257.958718:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829257.958765:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829257.958770:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829257.958773:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829258.982676:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829258.982681:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829258.982738:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829258.982742:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829258.982745:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829260.006768:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829260.006776:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829260.006880:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829260.006886:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829260.006891:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829261.030791:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829261.030797:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829261.030881:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829261.030887:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829261.030902:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787829261.158701:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829261.158710:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1787829261.158712:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787829261.158726:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787829261.158734:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787829261.158740:0:468:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 4061 00000100:00080000:2.0:1787829261.158748:0:468:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787829261.158751:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829261.158752:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787829261.158755:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787829261.158757:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787829261.158760:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:2.0:1787829261.158762:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:2.0:1787829261.158763:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829261.158764:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:2.0:1787829261.158768:0:468:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787829261.158770:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829261.158772:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787829261.158774:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787829261.158776:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829261.158778:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:2.0:1787829261.158780:0:468:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829261.158801:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787829261.158802:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829261.158805:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787829261.158807:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787829261.158811:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829261.158820:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787829261.158827:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2ed00 x1874670646812032/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787829261.158835:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787829261.158838:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829261.158851:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787829261.158864:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829261.158866:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1787829261.158867:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787829261.158868:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787829261.158870:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787829261.158872:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4061 00000100:00080000:2.0:1787829261.158874:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787829261.158876:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829261.158877:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787829261.158878:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787829261.158879:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787829261.158881:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000020:00000001:2.0:1787829261.158882:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:2.0:1787829261.158884:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829261.158884:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:2.0:1787829261.158886:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787829261.158888:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829261.158889:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787829261.158894:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787829261.158895:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829261.158896:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1787829261.158900:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1787829261.158902:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829261.158903:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829261.158905:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787829261.158906:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829261.158907:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787829261.158908:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787829261.158909:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829261.158912:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787829261.158916:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2f0c0 x1874670646812160/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787829261.158938:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787829261.158940:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829261.158945:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787829261.158948:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829261.158950:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1787829261.158952:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787829261.158954:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787829261.158956:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787829261.158957:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4061 00000100:00080000:2.0:1787829261.158960:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787829261.158962:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829261.158965:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:3.0:1787829261.158965:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:2.0:1787829261.158977:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787829261.158978:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787829261.158980:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000020:00000001:2.0:1787829261.158995:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:2.0:1787829261.158997:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829261.158999:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:3.0:1787829261.158999:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787829261.159006:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787829261.159011:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00080000:2.0:1787829261.159013:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:3.0:1787829261.159016:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829261.159023:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829261.159027:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787829261.159030:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787829261.159034:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787829261.159039:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829261.159040:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829261.159053:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829261.159059:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1787829261.159062:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1787829261.159078:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829261.159078:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646812032:192.168.203.143@tcp:250:kworker.0 00000100:00000001:2.0:1787829261.159086:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829261.159092:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787829261.159095:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787829261.159097:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829261.159099:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787829261.159099:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1787829261.159106:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787829261.159106:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829261.159115:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829261.159116:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:2.0:1787829261.159127:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829261.159146:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787829261.159147:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2c780 x1874670646812288/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787829261.159163:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787829261.159166:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829261.159172:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787829261.159175:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829261.159177:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1787829261.159178:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787829261.159180:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:3.0:1787829261.159182:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417879808 : -126234291671808 : ffff8d30c9e2ed00) 00000100:00080000:2.0:1787829261.159183:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787829261.159191:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:3.0:1787829261.159192:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00080000:2.0:1787829261.159207:0:468:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4061 00000100:00080000:2.0:1787829261.159211:0:468:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787829261.159214:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829261.159215:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787829261.159217:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787829261.159218:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787829261.159220:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000020:00000001:2.0:1787829261.159221:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:2.0:1787829261.159223:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829261.159224:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:2.0:1787829261.159227:0:468:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787829261.159229:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829261.159230:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787829261.159234:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787829261.159235:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829261.159237:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1787829261.159239:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000008:00000001:2.0:1787829261.159242:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829261.159244:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829261.159249:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787829261.159250:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829261.159251:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787829261.159252:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787829261.159254:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829261.159257:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787829261.159276:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2de00 x1874670646812416/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1787829261.159276:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:2.0:1787829261.159305:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787829261.159307:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829261.159308:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829261.159331:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787829261.159356:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829261.159359:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829261.159361:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1787829261.159369:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829261.159374:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4298735360 00000100:00000001:3.0:1787829261.159387:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 00000100:00000001:3.0:1787829261.159410:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4298735360 00000100:00000001:3.0:1787829261.159417:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 00000100:00000001:3.0:1787829261.159421:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829261.159426:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829261.159430:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1787829261.159442:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829261/real 1787829261] req@ffff8d30c9e2ed00 x1874670646812032/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829286 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1787829261.159461:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787829261.159464:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829261.159470:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829261.159474:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829261.159477:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829261.159480:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829261.159482:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829261.159488:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2ed00 x1874670646812032/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829286 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829261.159496:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829261.159499:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829261.159502:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829261.159504:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829261.159505:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829261.159508:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829261.159510:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829261.159513:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829261.159517:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829261.159535:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829261.159539:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646812032:192.168.203.143@tcp:250:kworker.0 00000100:00000001:3.0:1787829261.159544:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787829261.159545:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787829261.159547:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829261.159549:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787829261.159550:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829261.159552:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829261.159554:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646812160:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829261.159557:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1787829261.159559:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1787829261.159561:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829261.159562:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1787829261.159565:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829261.159571:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787829261.159572:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417880768 : -126234291670848 : ffff8d30c9e2f0c0) 00000100:00000001:3.0:1787829261.159577:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1787829261.159610:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:3.0:1787829261.159616:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787829261.159619:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829261.159622:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829261.159624:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1787829261.159628:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829261.159685:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298734103 00000100:00000001:3.0:1787829261.159688:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298735360 00000100:00000001:3.0:1787829261.159690:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298734103 00000100:00000001:3.0:1787829261.159692:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298735360 00000100:00000001:3.0:1787829261.159694:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829261.159695:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829261.159697:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1787829261.159700:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829261/real 1787829261] req@ffff8d30c9e2f0c0 x1874670646812160/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829286 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1787829261.159707:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787829261.159708:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829261.159710:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829261.159712:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829261.159713:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829261.159715:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829261.159716:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829261.159719:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2f0c0 x1874670646812160/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829286 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829261.159743:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829261.159746:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829261.159749:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829261.159751:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829261.159752:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829261.159755:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829261.159758:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829261.159761:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829261.159766:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829261.159769:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829261.159772:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646812160:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829261.159778:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829261.159781:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829261.159783:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829261.159786:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829261.159806:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829261.159808:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829261.159810:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829261.159814:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829261.159815:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829261.159821:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829261.159822:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829261.159823:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829261.159824:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829261.159825:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829261.159826:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829261.159827:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829261.159828:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829261.159829:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829261.159830:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829261.159831:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829261.159831:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829261.159833:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829261.159840:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787829261.159842:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829261.159843:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1787829261.159845:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787829261.159845:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787829261.159846:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787829261.159847:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829261.159849:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787829261.159850:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829261.159852:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829261.159872:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646812288:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829261.159876:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1787829261.159879:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1787829261.159882:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829261.159885:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1787829261.159888:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829261.159895:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787829261.159897:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417870208 : -126234291681408 : ffff8d30c9e2c780) 00000100:00000001:3.0:1787829261.159901:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1787829261.159917:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:3.0:1787829261.159923:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787829261.159925:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829261.159927:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829261.159929:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1787829261.159933:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829261.159937:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298734103 00000100:00000001:3.0:1787829261.159940:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298735360 00000100:00000001:3.0:1787829261.159943:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298734103 00000100:00000001:3.0:1787829261.159946:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298735360 00000100:00000001:3.0:1787829261.159949:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829261.159951:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829261.159954:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1787829261.159958:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829261/real 1787829261] req@ffff8d30c9e2c780 x1874670646812288/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829286 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1787829261.159968:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787829261.159969:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829261.159972:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829261.159974:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829261.159976:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829261.159978:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829261.159980:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829261.160001:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2c780 x1874670646812288/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829286 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829261.160018:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829261.160021:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829261.160024:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829261.160025:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829261.160027:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829261.160029:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829261.160030:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829261.160033:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829261.160036:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829261.160039:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829261.160042:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646812288:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829261.160046:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787829261.160063:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787829261.160064:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829261.160065:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787829261.160066:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829261.160067:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829261.160069:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646812416:192.168.203.143@tcp:8:kworker.0 00000100:00000001:3.0:1787829261.160071:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1787829261.160073:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1787829261.160073:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829261.160074:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1787829261.160076:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829261.160080:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787829261.160081:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417875968 : -126234291675648 : ffff8d30c9e2de00) 00000100:00000001:3.0:1787829261.160084:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1787829261.160090:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:3.0:1787829261.160092:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787829261.160094:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829261.160095:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829261.160096:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1787829261.160098:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829261.160100:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298734116 00000100:00000001:3.0:1787829261.160102:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298735360 00000100:00000001:3.0:1787829261.160104:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298734116 00000100:00000001:3.0:1787829261.160105:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298735360 00000100:00000001:3.0:1787829261.160106:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829261.160108:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829261.160109:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1787829261.160111:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829261/real 1787829261] req@ffff8d30c9e2de00 x1874670646812416/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829286 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1787829261.160115:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787829261.160116:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829261.160117:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829261.160118:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829261.160119:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829261.160120:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829261.160120:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829261.160123:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2de00 x1874670646812416/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829286 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829261.160127:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829261.160129:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829261.160130:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829261.160148:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829261.160149:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829261.160150:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829261.160151:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829261.160153:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829261.160155:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829261.160157:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829261.160160:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646812416:192.168.203.143@tcp:8:kworker.0 00000100:00000001:3.0:1787829261.160163:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829261.160164:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829261.160165:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829261.160166:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829261.160168:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829261.160168:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829261.160169:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829261.160171:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829261.160172:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829261.160174:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829261.160174:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829261.160176:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829261.160176:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829261.160177:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829261.160178:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829261.160179:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829261.160180:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829261.160182:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829261.160183:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829261.160187:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829261.160188:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829261.160191:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829261.160194:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787829261.160196:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829261.160215:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1787829261.160217:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829262.054784:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829262.054793:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829262.054915:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829262.054921:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829262.054926:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829263.078673:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829263.078678:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829263.078734:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829263.078738:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829263.078740:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829264.102713:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829264.102720:0:815:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1787829264.102722:0:815:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1787829264.102724:0:815:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829264.102728:0:815:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1787829264.102730:0:815:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829264.102731:0:815:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829264.102733:0:815:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829264.102866:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829264.102872:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829264.102877:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829265.126688:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829265.126695:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829265.126786:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829265.126790:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829265.126793:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829266.150717:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829266.150725:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829266.150780:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829266.150785:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829266.150799:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787829266.278713:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829266.278722:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1787829266.278724:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787829266.278728:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787829266.278732:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787829266.278738:0:468:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 4066 00000100:00080000:2.0:1787829266.278748:0:468:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787829266.278751:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829266.278752:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787829266.278756:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787829266.278759:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787829266.278761:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:2.0:1787829266.278763:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:2.0:1787829266.278764:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829266.278766:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:2.0:1787829266.278769:0:468:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787829266.278771:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829266.278773:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787829266.278776:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787829266.278778:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829266.278780:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:2.0:1787829266.278781:0:468:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829266.278791:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787829266.278792:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829266.278794:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787829266.278796:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787829266.278800:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829266.278809:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787829266.278816:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2c3c0 x1874670646812544/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787829266.278824:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787829266.278827:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829266.278838:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787829266.278840:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829266.278843:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1787829266.278844:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787829266.278845:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787829266.278847:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787829266.278848:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4066 00000100:00080000:2.0:1787829266.278850:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787829266.278851:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829266.278852:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787829266.278853:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787829266.278854:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787829266.278855:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000020:00000001:2.0:1787829266.278857:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:2.0:1787829266.278859:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829266.278860:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:2.0:1787829266.278862:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787829266.278864:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829266.278866:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787829266.278870:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787829266.278871:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829266.278873:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1787829266.278878:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1787829266.278880:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829266.278881:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829266.278884:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787829266.278886:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829266.278887:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787829266.278888:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787829266.278890:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829266.278893:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787829266.278896:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2fc00 x1874670646812672/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787829266.278898:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787829266.278906:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787829266.278907:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1787829266.278908:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829266.278912:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1787829266.278913:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829266.278918:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00080000:2.0:1787829266.278919:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1787829266.278922:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787829266.278924:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1787829266.278931:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787829266.278932:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 02000000:00000001:3.0:1787829266.278934:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:2.0:1787829266.278936:0:468:0:(import.c:485:import_select_connection()) Process entered 02000000:00000001:3.0:1787829266.278939:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00080000:2.0:1787829266.278940:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4066 00000100:00080000:2.0:1787829266.278948:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 02000000:00000001:3.0:1787829266.278948:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829266.278955:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829266.278958:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787829266.278963:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000100:00100000:3.0:1787829266.278963:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646812544:192.168.203.143@tcp:250:kworker.0 00000020:00000001:2.0:1787829266.278968:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787829266.278972:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:3.0:1787829266.278976:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000020:00000001:2.0:1787829266.278977:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:2.0:1787829266.278981:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829266.278982:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 02000000:00000001:3.0:1787829266.278984:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000100:00080000:2.0:1787829266.278988:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 02000000:00000001:3.0:1787829266.278989:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829266.278991:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829266.278994:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1787829266.278995:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787829266.278998:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787829266.279001:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829266.279003:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829266.279005:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1787829266.279011:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1787829266.279015:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829266.279016:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829266.279019:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787829266.279020:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829266.279021:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787829266.279022:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787829266.279024:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829266.279027:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829266.279029:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787829266.279033:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417869248 : -126234291682368 : ffff8d30c9e2c3c0) 00000100:00100000:2.0:1787829266.279034:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2d2c0 x1874670646812800/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787829266.279043:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:2.0:1787829266.279050:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787829266.279051:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829266.279054:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787829266.279055:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829266.279058:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1787829266.279059:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787829266.279060:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787829266.279063:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787829266.279065:0:468:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4066 00000100:00080000:2.0:1787829266.279077:0:468:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787829266.279079:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829266.279080:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787829266.279081:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787829266.279082:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787829266.279083:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000020:00000001:2.0:1787829266.279085:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:2.0:1787829266.279087:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829266.279088:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:2.0:1787829266.279090:0:468:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000800:00000100:3.0:1787829266.279098:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:2.0:1787829266.279100:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829266.279103:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787829266.279106:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787829266.279107:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829266.279109:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1787829266.279111:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000008:00000001:2.0:1787829266.279112:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829266.279114:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829266.279116:0:2390:0:(events.c:34:request_out_callback()) Process entered 02000000:00000001:2.0:1787829266.279118:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787829266.279120:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829266.279121:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:3.0:1787829266.279122:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 02000000:00000001:2.0:1787829266.279123:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:3.0:1787829266.279126:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829266.279127:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829266.279133:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1787829266.279136:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829266.279140:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787829266.279147:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2cb40 x1874670646812928/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787829266.279149:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4298740480 00000100:00000001:2.0:1787829266.279160:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787829266.279162:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829266.279164:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829266.279165:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 00000100:00000001:3.0:1787829266.279178:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4298740480 00000100:00000001:3.0:1787829266.279184:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 00000100:00000001:3.0:1787829266.279193:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829266.279195:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829266.279197:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1787829266.279204:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829266/real 1787829266] req@ffff8d30c9e2c3c0 x1874670646812544/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829291 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1787829266.279213:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787829266.279215:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829266.279217:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829266.279220:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829266.279221:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829266.279223:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829266.279225:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829266.279229:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2c3c0 x1874670646812544/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829291 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829266.279235:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829266.279238:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829266.279240:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829266.279241:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829266.279242:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829266.279244:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829266.279257:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829266.279260:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829266.279264:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829266.279265:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829266.279268:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646812544:192.168.203.143@tcp:250:kworker.0 00000100:00000001:3.0:1787829266.279272:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829266.279274:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829266.279276:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829266.279278:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829266.279281:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829266.279284:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829266.279286:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829266.279290:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829266.279292:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829266.279294:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829266.279295:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829266.279297:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829266.279305:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787829266.279307:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829266.279309:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1787829266.279310:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787829266.279312:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787829266.279313:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787829266.279314:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829266.279316:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787829266.279318:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829266.279320:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829266.279322:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646812672:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829266.279325:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1787829266.279327:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1787829266.279328:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829266.279329:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1787829266.279331:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829266.279336:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787829266.279337:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417883648 : -126234291667968 : ffff8d30c9e2fc00) 00000100:00000001:3.0:1787829266.279341:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1787829266.279358:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:3.0:1787829266.279361:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787829266.279363:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829266.279364:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829266.279365:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1787829266.279367:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829266.279371:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298739222 00000100:00000001:3.0:1787829266.279373:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298740480 00000100:00000001:3.0:1787829266.279375:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298739222 00000100:00000001:3.0:1787829266.279377:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298740480 00000100:00000001:3.0:1787829266.279378:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829266.279380:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829266.279381:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1787829266.279383:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829266/real 1787829266] req@ffff8d30c9e2fc00 x1874670646812672/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829291 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1787829266.279388:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787829266.279389:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829266.279391:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829266.279392:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829266.279393:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829266.279394:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829266.279395:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829266.279398:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2fc00 x1874670646812672/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829291 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829266.279403:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829266.279405:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829266.279413:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829266.279414:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829266.279415:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829266.279416:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829266.279420:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829266.279422:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829266.279424:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829266.279426:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829266.279428:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646812672:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829266.279431:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787829266.279432:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787829266.279433:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829266.279434:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787829266.279435:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829266.279437:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829266.279439:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2d2c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646812800:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829266.279441:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1787829266.279443:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1787829266.279444:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829266.279445:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1787829266.279447:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829266.279450:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787829266.279451:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417873088 : -126234291678528 : ffff8d30c9e2d2c0) 00000100:00000001:3.0:1787829266.279454:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1787829266.279460:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:3.0:1787829266.279463:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787829266.279464:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829266.279466:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829266.279467:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1787829266.279469:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829266.279471:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298739222 00000100:00000001:3.0:1787829266.279473:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298740480 00000100:00000001:3.0:1787829266.279474:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298739222 00000100:00000001:3.0:1787829266.279476:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298740480 00000100:00000001:3.0:1787829266.279478:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829266.279479:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829266.279480:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1787829266.279482:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829266/real 1787829266] req@ffff8d30c9e2d2c0 x1874670646812800/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829291 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1787829266.279486:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787829266.279487:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829266.279489:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829266.279490:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829266.279491:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829266.279492:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829266.279493:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829266.279495:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2d2c0 x1874670646812800/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829291 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829266.279499:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829266.279501:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829266.279502:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829266.279503:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829266.279503:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829266.279505:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829266.279505:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829266.279507:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829266.279509:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829266.279510:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829266.279513:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2d2c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646812800:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829266.279515:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787829266.279516:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787829266.279518:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829266.279519:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787829266.279520:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829266.279521:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829266.279523:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646812928:192.168.203.143@tcp:8:kworker.0 00000100:00000001:3.0:1787829266.279525:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1787829266.279526:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1787829266.279527:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829266.279528:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1787829266.279529:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829266.279533:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787829266.279533:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417871168 : -126234291680448 : ffff8d30c9e2cb40) 00000100:00000001:3.0:1787829266.279535:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1787829266.279541:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:3.0:1787829266.279543:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787829266.279544:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829266.279545:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829266.279546:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1787829266.279548:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829266.279550:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298739235 00000100:00000001:3.0:1787829266.279552:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298740480 00000100:00000001:3.0:1787829266.279553:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298739235 00000100:00000001:3.0:1787829266.279555:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298740480 00000100:00000001:3.0:1787829266.279556:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829266.279557:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829266.279558:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1787829266.279561:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829266/real 1787829266] req@ffff8d30c9e2cb40 x1874670646812928/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829291 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1787829266.279570:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787829266.279571:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829266.279572:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829266.279573:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829266.279574:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829266.279575:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829266.279576:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829266.279578:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2cb40 x1874670646812928/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829291 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829266.279582:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829266.279583:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829266.279584:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829266.279585:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829266.279586:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829266.279588:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829266.279589:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829266.279591:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829266.279593:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829266.279594:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829266.279596:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646812928:192.168.203.143@tcp:8:kworker.0 00000100:00000001:3.0:1787829266.279598:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829266.279630:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829266.279631:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829266.279632:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829266.279634:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829266.279634:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829266.279635:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829266.279636:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829266.279637:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829266.279640:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829266.279641:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829266.279642:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829266.279643:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829266.279644:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829266.279645:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829266.279646:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829266.279647:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829266.279648:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829266.279649:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829266.279652:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829266.279652:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829266.279653:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829266.279654:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829266.279655:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829266.279656:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829266.279656:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829266.279657:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829266.279658:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829266.279658:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829266.279659:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829266.279660:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829266.279661:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829266.279663:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787829266.279664:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829266.279665:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1787829266.279667:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829267.174715:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829267.174720:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829267.174774:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829267.174779:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829267.174783:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829268.198709:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829268.198731:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829268.198793:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829268.198796:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829268.198799:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829269.222705:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829269.222710:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829269.222745:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829269.222750:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829269.222753:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829270.246685:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829270.246692:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829270.246756:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829270.246761:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829270.246765:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829271.270674:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829271.270678:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829271.270730:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829271.270733:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829271.270736:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787829271.398739:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829271.398750:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1787829271.398753:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787829271.398757:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787829271.398761:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787829271.398768:0:40798:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 4071 00000100:00080000:0.0:1787829271.398778:0:40798:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787829271.398782:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829271.398784:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787829271.398788:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787829271.398791:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787829271.398794:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:0.0:1787829271.398797:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:0.0:1787829271.398799:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829271.398800:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:0.0:1787829271.398806:0:40798:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787829271.398809:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829271.398812:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787829271.398815:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787829271.398819:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829271.398821:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1787829271.398824:0:40798:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829271.398834:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787829271.398837:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829271.398849:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787829271.398852:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787829271.398856:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829271.398869:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787829271.398878:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d26940 x1874670646813056/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787829271.398890:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787829271.398894:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829271.398914:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787829271.398917:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829271.398920:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1787829271.398922:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787829271.398924:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787829271.398928:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787829271.398931:0:40798:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4071 00000100:00080000:0.0:1787829271.398935:0:40798:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787829271.398938:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829271.398940:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787829271.398943:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787829271.398945:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787829271.398947:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000020:00000001:0.0:1787829271.398949:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:3.0:1787829271.398951:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787829271.398952:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829271.398957:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:3.0:1787829271.398957:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787829271.398960:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00080000:0.0:1787829271.398964:0:40798:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:3.0:1787829271.398965:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787829271.398967:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829271.398970:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829271.398972:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787829271.398975:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787829271.398976:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829271.398980:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829271.398983:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00100000:3.0:1787829271.398985:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d26940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646813056:192.168.203.143@tcp:250:kworker.0 02000000:00000001:0.0:1787829271.398990:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829271.398991:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1787829271.398995:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:0.0:1787829271.398996:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 02000000:00000001:3.0:1787829271.398997:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829271.399000:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000008:00100000:0.0:1787829271.399002:0:40798:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 02000000:00000001:3.0:1787829271.399005:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:0.0:1787829271.399007:0:40798:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829271.399012:0:40798:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829271.399017:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787829271.399019:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829271.399021:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787829271.399022:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:3.0:1787829271.399024:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601346880 : -126234108204736 : ffff8d30d4d26940) 02000000:00000001:0.0:1787829271.399026:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:3.0:1787829271.399028:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 02000000:00000001:0.0:1787829271.399030:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829271.399038:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787829271.399045:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d25680 x1874670646813184/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787829271.399054:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787829271.399057:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:3.0:1787829271.399061:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:0.0:1787829271.399064:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787829271.399069:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829271.399072:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1787829271.399073:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1787829271.399075:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:3.0:1787829271.399078:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00080000:0.0:1787829271.399079:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787829271.399081:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829271.399085:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:3.0:1787829271.399085:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1787829271.399089:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787829271.399092:0:40798:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4071 00000100:00000001:3.0:1787829271.399094:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4298745600 00000100:00080000:0.0:1787829271.399098:0:40798:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787829271.399102:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829271.399104:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1787829271.399105:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787829271.399110:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:3.0:1787829271.399111:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4298745600 00000020:00000001:0.0:1787829271.399112:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:3.0:1787829271.399116:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 00000020:00000001:0.0:1787829271.399117:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:3.0:1787829271.399121:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000020:00000001:0.0:1787829271.399122:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:3.0:1787829271.399124:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829271.399129:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829271.399129:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00000001:0.0:1787829271.399133:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00100000:3.0:1787829271.399136:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829271/real 1787829271] req@ffff8d30d4d26940 x1874670646813056/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829296 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829271.399141:0:40798:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787829271.399148:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829271.399153:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000400:00000001:3.0:1787829271.399154:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787829271.399156:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 02000000:00000001:0.0:1787829271.399158:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829271.399159:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829271.399163:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 02000000:00000001:0.0:1787829271.399164:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829271.399165:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829271.399168:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:3.0:1787829271.399169:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829271.399172:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000008:00100000:0.0:1787829271.399173:0:40798:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787829271.399177:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d26940 x1874670646813056/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829296 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00000001:0.0:1787829271.399179:0:40798:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829271.399184:0:40798:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829271.399191:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:0.0:1787829271.399194:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:3.0:1787829271.399194:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 02000000:00000001:0.0:1787829271.399198:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829271.399198:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829271.399203:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787829271.399205:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:3.0:1787829271.399205:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:0.0:1787829271.399209:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:3.0:1787829271.399222:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829271.399224:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829271.399227:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 02000000:00000001:0.0:1787829271.399229:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787829271.399232:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829271.399235:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000001:0.0:1787829271.399238:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829271.399239:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d26940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646813056:192.168.203.143@tcp:250:kworker.0 00000100:00000001:3.0:1787829271.399246:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787829271.399249:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d252c0 x1874670646813312/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787829271.399250:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829271.399252:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829271.399255:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829271.399259:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829271.399261:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000100:00000001:0.0:1787829271.399265:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000020:00000001:3.0:1787829271.399265:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 00000100:00000001:0.0:1787829271.399271:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829271.399271:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829271.399273:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787829271.399278:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829271.399278:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829271.399279:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829271.399285:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:0.0:1787829271.399286:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829271.399291:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1787829271.399292:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787829271.399293:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:3.0:1787829271.399296:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787829271.399298:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787829271.399301:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787829271.399303:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:3.0:1787829271.399307:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00080000:0.0:1787829271.399309:0:40798:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4071 00000100:00000001:3.0:1787829271.399310:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787829271.399313:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00080000:0.0:1787829271.399317:0:40798:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787829271.399317:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829271.399321:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:0.0:1787829271.399323:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 02000000:00000001:3.0:1787829271.399325:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787829271.399327:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 02000000:00000001:3.0:1787829271.399330:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787829271.399332:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787829271.399335:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00100000:3.0:1787829271.399337:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d25680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646813184:192.168.203.143@tcp:38:kworker.0 00000020:00000001:0.0:1787829271.399338:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000020:00000001:0.0:1787829271.399344:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:3.0:1787829271.399345:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:0.0:1787829271.399351:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 02000000:00000001:3.0:1787829271.399351:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1787829271.399354:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829271.399355:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 02000000:00000001:3.0:1787829271.399359:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00080000:0.0:1787829271.399364:0:40798:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 02000000:00000001:3.0:1787829271.399365:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829271.399369:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829271.399373:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:3.0:1787829271.399374:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 02000000:00000001:0.0:1787829271.399377:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829271.399378:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601342080 : -126234108209536 : ffff8d30d4d25680) 02000000:00000001:0.0:1787829271.399383:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829271.399386:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:3.0:1787829271.399387:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000008:00100000:0.0:1787829271.399389:0:40798:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1787829271.399393:0:40798:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829271.399395:0:40798:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829271.399400:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787829271.399402:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829271.399404:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787829271.399406:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787829271.399408:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:3.0:1787829271.399413:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:0.0:1787829271.399415:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829271.399420:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00100000:0.0:1787829271.399425:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d27c00 x1874670646813440/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787829271.399425:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829271.399429:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829271.399432:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1787829271.399436:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829271.399442:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787829271.399444:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298744342 00000100:00000001:0.0:1787829271.399446:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829271.399451:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298745600 00000100:00000001:0.0:1787829271.399453:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829271.399456:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298744342 00000100:00000001:3.0:1787829271.399460:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298745600 00000100:00000001:3.0:1787829271.399462:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829271.399465:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829271.399467:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1787829271.399473:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829271/real 1787829271] req@ffff8d30d4d25680 x1874670646813184/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829296 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1787829271.399488:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787829271.399489:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829271.399492:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829271.399493:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829271.399494:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829271.399496:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829271.399497:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829271.399500:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d25680 x1874670646813184/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829296 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829271.399505:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829271.399506:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829271.399508:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829271.399509:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829271.399510:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829271.399511:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829271.399512:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829271.399514:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829271.399516:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829271.399517:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829271.399519:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d25680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646813184:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829271.399522:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787829271.399523:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787829271.399524:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829271.399526:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787829271.399527:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829271.399528:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829271.399530:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d252c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646813312:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829271.399532:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1787829271.399534:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1787829271.399535:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829271.399536:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1787829271.399538:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829271.399541:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787829271.399542:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601341120 : -126234108210496 : ffff8d30d4d252c0) 00000100:00000001:3.0:1787829271.399545:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1787829271.399554:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:3.0:1787829271.399557:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787829271.399564:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829271.399566:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829271.399567:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1787829271.399569:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829271.399571:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298744342 00000100:00000001:3.0:1787829271.399573:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298745600 00000100:00000001:3.0:1787829271.399575:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298744342 00000100:00000001:3.0:1787829271.399576:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298745600 00000100:00000001:3.0:1787829271.399578:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829271.399579:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829271.399580:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1787829271.399582:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829271/real 1787829271] req@ffff8d30d4d252c0 x1874670646813312/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829296 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1787829271.399587:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787829271.399588:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829271.399589:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829271.399590:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829271.399591:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829271.399592:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829271.399593:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829271.399596:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d252c0 x1874670646813312/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829296 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829271.399629:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829271.399631:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829271.399633:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829271.399634:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829271.399635:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829271.399636:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829271.399637:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829271.399639:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829271.399641:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829271.399642:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829271.399645:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d252c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646813312:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829271.399647:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829271.399649:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829271.399650:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829271.399651:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829271.399653:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829271.399654:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829271.399654:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829271.399656:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829271.399657:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829271.399658:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829271.399659:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829271.399660:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829271.399661:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829271.399662:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829271.399663:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829271.399664:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829271.399665:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829271.399666:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829271.399666:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829271.399667:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829271.399668:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829271.399669:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829271.399671:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787829271.399672:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829271.399673:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1787829271.399674:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787829271.399675:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787829271.399676:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787829271.399677:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829271.399678:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787829271.399679:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829271.399681:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829271.399682:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d27c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646813440:192.168.203.143@tcp:8:kworker.0 00000100:00000001:3.0:1787829271.399685:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1787829271.399686:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1787829271.399687:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829271.399688:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1787829271.399690:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829271.399693:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787829271.399694:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601351680 : -126234108199936 : ffff8d30d4d27c00) 00000100:00000001:3.0:1787829271.399697:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1787829271.399705:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:3.0:1787829271.399707:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787829271.399709:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829271.399710:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829271.399711:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1787829271.399713:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829271.399715:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298744356 00000100:00000001:3.0:1787829271.399716:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298745600 00000100:00000001:3.0:1787829271.399718:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298744356 00000100:00000001:3.0:1787829271.399719:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298745600 00000100:00000001:3.0:1787829271.399721:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829271.399722:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829271.399723:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1787829271.399725:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829271/real 1787829271] req@ffff8d30d4d27c00 x1874670646813440/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829296 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1787829271.399730:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787829271.399731:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829271.399732:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829271.399733:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829271.399734:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829271.399736:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829271.399737:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829271.399739:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d27c00 x1874670646813440/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829296 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829271.399743:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829271.399744:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829271.399745:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829271.399746:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829271.399748:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829271.399749:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829271.399750:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829271.399752:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829271.399754:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829271.399755:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829271.399757:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d27c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646813440:192.168.203.143@tcp:8:kworker.0 00000100:00000001:3.0:1787829271.399759:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829271.399760:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829271.399760:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829271.399761:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829271.399762:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829271.399763:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829271.399764:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829271.399765:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829271.399766:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829271.399767:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829271.399768:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829271.399769:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829271.399771:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787829271.399772:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829271.399773:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1787829271.399774:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829272.294699:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829272.294705:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829272.294760:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829272.294764:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829272.294767:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829273.318664:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829273.318671:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829273.318766:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829273.318769:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829273.318772:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829274.342694:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829274.342701:0:815:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1787829274.342703:0:815:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1787829274.342707:0:815:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829274.342712:0:815:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1787829274.342714:0:815:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829274.342716:0:815:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829274.342719:0:815:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829274.342770:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829274.342773:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829274.342776:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829275.366703:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829275.366708:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829275.366758:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829275.366762:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829275.366765:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829276.390700:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829276.390706:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829276.390778:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829276.390797:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829276.390800:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787829276.518673:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829276.518682:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1787829276.518685:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787829276.518688:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787829276.518693:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787829276.518700:0:40798:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 4077 00000100:00080000:0.0:1787829276.518710:0:40798:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787829276.518713:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829276.518716:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787829276.518720:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787829276.518723:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787829276.518726:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:0.0:1787829276.518728:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:0.0:1787829276.518731:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829276.518732:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:0.0:1787829276.518737:0:40798:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787829276.518739:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829276.518742:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787829276.518745:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787829276.518749:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829276.518751:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1787829276.518753:0:40798:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829276.518764:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787829276.518765:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829276.518768:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787829276.518771:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787829276.518775:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829276.518785:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787829276.518794:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d270c0 x1874670646813568/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787829276.518806:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787829276.518810:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829276.518826:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787829276.518829:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829276.518832:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1787829276.518834:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787829276.518836:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787829276.518839:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787829276.518841:0:40798:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4077 00000100:00080000:0.0:1787829276.518845:0:40798:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787829276.518848:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829276.518849:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787829276.518852:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787829276.518854:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787829276.518856:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000020:00000001:0.0:1787829276.518859:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:0.0:1787829276.518861:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829276.518863:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:0.0:1787829276.518866:0:40798:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:3.0:1787829276.518868:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787829276.518869:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829276.518873:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:3.0:1787829276.518876:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 02000000:00000001:0.0:1787829276.518879:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829276.518880:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 02000000:00000001:0.0:1787829276.518884:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829276.518886:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787829276.518887:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:3.0:1787829276.518891:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000008:00100000:0.0:1787829276.518893:0:40798:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1787829276.518898:0:40798:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829276.518898:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:0.0:1787829276.518902:0:40798:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829276.518903:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787829276.518907:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787829276.518907:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829276.518911:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829276.518915:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00100000:3.0:1787829276.518916:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d270c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646813568:192.168.203.143@tcp:250:kworker.0 02000000:00000001:0.0:1787829276.518917:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787829276.518921:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829276.518922:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1787829276.518927:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:0.0:1787829276.518928:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829276.518929:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829276.518934:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00100000:0.0:1787829276.518938:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d25a40 x1874670646813696/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:3.0:1787829276.518939:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829276.518950:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787829276.518952:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829276.518954:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787829276.518956:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829276.518957:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601348800 : -126234108202816 : ffff8d30d4d270c0) 00000100:00080000:0.0:1787829276.518963:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1787829276.518964:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00080000:0.0:1787829276.518970:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1787829276.518972:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787829276.518974:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787829276.518976:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787829276.518979:0:40798:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4077 00000100:00080000:0.0:1787829276.518983:0:40798:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787829276.518985:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829276.518986:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787829276.518989:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787829276.518990:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787829276.518992:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000020:00000001:0.0:1787829276.518995:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:0.0:1787829276.518997:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829276.518999:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000800:00000100:3.0:1787829276.519004:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00080000:0.0:1787829276.519006:0:40798:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787829276.519010:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829276.519012:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787829276.519014:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787829276.519017:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829276.519020:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:3.0:1787829276.519020:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000008:00100000:0.0:1787829276.519025:0:40798:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1787829276.519028:0:40798:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829276.519028:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787829276.519032:0:40798:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829276.519033:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829276.519036:0:2390:0:(events.c:68:request_out_callback()) Process leaving 02000000:00000001:0.0:1787829276.519037:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787829276.519040:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829276.519041:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829276.519042:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787829276.519046:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:3.0:1787829276.519047:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4298750720 02000000:00000001:0.0:1787829276.519048:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829276.519056:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829276.519058:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 00000100:00100000:0.0:1787829276.519063:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d24f00 x1874670646813824/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787829276.519066:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4298750720 00000100:00000001:3.0:1787829276.519073:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 00000100:00000001:3.0:1787829276.519078:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829276.519083:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829276.519088:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00000001:0.0:1787829276.519090:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787829276.519093:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829276.519095:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829276/real 1787829276] req@ffff8d30d4d270c0 x1874670646813568/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829301 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787829276.519097:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787829276.519103:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829276.519111:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1787829276.519114:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000400:00000001:3.0:1787829276.519115:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00080000:0.0:1787829276.519116:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000400:00000001:3.0:1787829276.519119:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787829276.519121:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:3.0:1787829276.519123:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00080000:0.0:1787829276.519127:0:40798:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4077 00000100:00000001:3.0:1787829276.519127:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829276.519131:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:0.0:1787829276.519133:0:40798:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787829276.519134:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787829276.519137:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829276.519137:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829276.519141:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00100000:3.0:1787829276.519144:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d270c0 x1874670646813568/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829301 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:0.0:1787829276.519145:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787829276.519148:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787829276.519151:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000020:00000001:0.0:1787829276.519157:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:3.0:1787829276.519158:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829276.519162:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829276.519163:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787829276.519165:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:0.0:1787829276.519169:0:40798:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787829276.519172:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829276.519189:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:0.0:1787829276.519194:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:3.0:1787829276.519194:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829276.519197:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:0.0:1787829276.519198:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829276.519199:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829276.519202:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 02000000:00000001:0.0:1787829276.519203:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787829276.519206:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00000001:0.0:1787829276.519208:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1787829276.519211:0:40798:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00080000:3.0:1787829276.519212:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829276.519217:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000008:00000001:0.0:1787829276.519218:0:40798:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829276.519223:0:40798:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829276.519224:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d270c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646813568:192.168.203.143@tcp:250:kworker.0 02000000:00000001:0.0:1787829276.519231:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:3.0:1787829276.519233:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:0.0:1787829276.519234:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829276.519239:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:3.0:1787829276.519239:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 02000000:00000001:0.0:1787829276.519243:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:3.0:1787829276.519244:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787829276.519245:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829276.519249:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 00000100:00000001:0.0:1787829276.519251:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829276.519254:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829276.519259:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000100:00100000:0.0:1787829276.519260:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d24000 x1874670646813952/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000020:00000001:3.0:1787829276.519262:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829276.519268:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 00000100:00000001:0.0:1787829276.519272:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 02000000:00000001:3.0:1787829276.519272:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787829276.519277:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829276.519279:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787829276.519280:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829276.519282:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829276.519285:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829276.519291:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787829276.519293:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829276.519294:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1787829276.519296:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787829276.519297:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787829276.519299:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787829276.519300:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829276.519302:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787829276.519304:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829276.519307:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829276.519312:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d25a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646813696:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829276.519316:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1787829276.519318:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1787829276.519319:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829276.519321:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1787829276.519324:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829276.519331:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787829276.519332:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601343040 : -126234108208576 : ffff8d30d4d25a40) 00000100:00000001:3.0:1787829276.519337:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1787829276.519351:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:3.0:1787829276.519356:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787829276.519358:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829276.519359:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829276.519361:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1787829276.519363:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829276.519366:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298749462 00000100:00000001:3.0:1787829276.519368:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298750720 00000100:00000001:3.0:1787829276.519370:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298749462 00000100:00000001:3.0:1787829276.519371:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298750720 00000100:00000001:3.0:1787829276.519373:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829276.519374:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829276.519376:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1787829276.519381:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829276/real 1787829276] req@ffff8d30d4d25a40 x1874670646813696/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829301 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1787829276.519390:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787829276.519391:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829276.519394:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829276.519395:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829276.519397:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829276.519399:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829276.519400:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829276.519404:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d25a40 x1874670646813696/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829301 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829276.519413:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829276.519416:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829276.519419:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829276.519420:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829276.519422:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829276.519424:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829276.519426:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829276.519428:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829276.519430:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829276.519432:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829276.519434:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d25a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646813696:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829276.519437:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787829276.519438:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787829276.519439:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829276.519441:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787829276.519443:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829276.519446:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829276.519449:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d24f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646813824:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829276.519452:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1787829276.519454:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1787829276.519456:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829276.519457:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1787829276.519461:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829276.519466:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787829276.519467:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601340160 : -126234108211456 : ffff8d30d4d24f00) 00000100:00000001:3.0:1787829276.519471:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1787829276.519485:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:3.0:1787829276.519490:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787829276.519493:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829276.519494:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829276.519496:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1787829276.519499:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829276.519503:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298749462 00000100:00000001:3.0:1787829276.519505:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298750720 00000100:00000001:3.0:1787829276.519508:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298749462 00000100:00000001:3.0:1787829276.519510:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298750720 00000100:00000001:3.0:1787829276.519513:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829276.519515:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829276.519517:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1787829276.519522:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829276/real 1787829276] req@ffff8d30d4d24f00 x1874670646813824/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829301 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1787829276.519530:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787829276.519532:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829276.519535:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829276.519537:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829276.519538:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829276.519539:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829276.519540:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829276.519543:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d24f00 x1874670646813824/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829301 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829276.519548:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829276.519549:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829276.519551:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829276.519552:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829276.519553:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829276.519554:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829276.519555:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829276.519557:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829276.519559:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829276.519560:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829276.519563:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d24f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646813824:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829276.519565:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787829276.519566:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787829276.519567:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829276.519569:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787829276.519570:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829276.519571:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829276.519573:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d24000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646813952:192.168.203.143@tcp:8:kworker.0 00000100:00000001:3.0:1787829276.519575:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1787829276.519577:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1787829276.519578:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829276.519579:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1787829276.519581:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829276.519585:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787829276.519586:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601336320 : -126234108215296 : ffff8d30d4d24000) 00000100:00000001:3.0:1787829276.519589:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1787829276.519597:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:3.0:1787829276.519627:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787829276.519630:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829276.519631:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829276.519633:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1787829276.519636:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829276.519639:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298749476 00000100:00000001:3.0:1787829276.519642:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298750720 00000100:00000001:3.0:1787829276.519645:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298749476 00000100:00000001:3.0:1787829276.519647:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298750720 00000100:00000001:3.0:1787829276.519650:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829276.519652:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829276.519654:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1787829276.519659:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829276/real 1787829276] req@ffff8d30d4d24000 x1874670646813952/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829301 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1787829276.519673:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787829276.519674:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829276.519676:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829276.519677:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829276.519678:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829276.519680:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829276.519681:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829276.519683:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d24000 x1874670646813952/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829301 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829276.519688:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829276.519690:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829276.519691:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829276.519692:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829276.519693:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829276.519694:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829276.519695:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829276.519697:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829276.519698:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829276.519700:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829276.519701:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d24000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646813952:192.168.203.143@tcp:8:kworker.0 00000100:00000001:3.0:1787829276.519704:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829276.519705:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829276.519706:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829276.519707:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829276.519709:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829276.519710:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829276.519711:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829276.519712:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829276.519713:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829276.519715:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829276.519716:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829276.519717:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829276.519718:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829276.519719:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829276.519720:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829276.519720:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829276.519721:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829276.519722:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829276.519723:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829276.519725:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829276.519726:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829276.519727:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829276.519727:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829276.519728:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829276.519729:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829276.519729:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829276.519730:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829276.519731:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829276.519732:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829276.519732:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829276.519733:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829276.519734:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829276.519737:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787829276.519738:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829276.519739:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1787829276.519740:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829277.414702:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829277.414706:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829277.414784:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829277.414788:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829277.414791:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829278.438674:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829278.438679:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829278.438728:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829278.438732:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829278.438735:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829279.462725:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829279.462731:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829279.462793:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829279.462798:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829279.462803:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829280.486679:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829280.486685:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829280.486735:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829280.486738:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829280.486741:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829281.510696:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829281.510701:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829281.510765:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829281.510768:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829281.510771:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787829281.638724:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829281.638734:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1787829281.638736:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787829281.638740:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787829281.638756:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787829281.638762:0:468:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 4082 00000100:00080000:2.0:1787829281.638772:0:468:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787829281.638775:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829281.638776:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787829281.638780:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787829281.638782:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787829281.638785:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:2.0:1787829281.638786:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:2.0:1787829281.638788:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829281.638789:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:2.0:1787829281.638793:0:468:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787829281.638795:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829281.638797:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787829281.638800:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787829281.638802:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829281.638804:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:2.0:1787829281.638806:0:468:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829281.638827:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787829281.638828:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829281.638831:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787829281.638833:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787829281.638837:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829281.638847:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787829281.638855:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2e1c0 x1874670646814080/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787829281.638863:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787829281.638867:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829281.638881:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787829281.638883:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829281.638886:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1787829281.638888:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787829281.638889:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787829281.638892:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787829281.638893:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4082 00000100:00080000:2.0:1787829281.638896:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787829281.638898:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829281.638899:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787829281.638901:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787829281.638902:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787829281.638904:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000020:00000001:2.0:1787829281.638906:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:2.0:1787829281.638908:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829281.638909:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:2.0:1787829281.638912:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787829281.638913:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829281.638915:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787829281.638920:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829281.638920:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:2.0:1787829281.638925:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829281.638925:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1787829281.638927:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:3.0:1787829281.638930:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000008:00100000:2.0:1787829281.638932:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00000001:3.0:1787829281.638933:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787829281.638937:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829281.638943:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787829281.638945:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000008:00000001:2.0:1787829281.638946:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829281.638948:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829281.638950:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829281.638957:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646814080:192.168.203.143@tcp:250:kworker.0 02000000:00000001:2.0:1787829281.638959:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787829281.638964:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829281.638965:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:2.0:1787829281.638966:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787829281.638969:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787829281.638970:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1787829281.638972:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829281.638974:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829281.638977:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:2.0:1787829281.638980:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829281.638982:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787829281.638987:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2d680 x1874670646814208/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787829281.638994:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787829281.638996:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829281.639006:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829281.639008:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00080000:2.0:1787829281.639009:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1787829281.639011:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417876928 : -126234291674688 : ffff8d30c9e2e1c0) 00000100:00000001:3.0:1787829281.639016:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00080000:2.0:1787829281.639021:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1787829281.639023:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787829281.639024:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787829281.639029:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787829281.639031:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4082 00000100:00080000:2.0:1787829281.639038:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787829281.639041:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829281.639042:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787829281.639044:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787829281.639045:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787829281.639047:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000020:00000001:2.0:1787829281.639049:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000800:00000100:3.0:1787829281.639053:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:2.0:1787829281.639054:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829281.639059:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:2.0:1787829281.639074:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787829281.639076:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829281.639078:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:3.0:1787829281.639080:0:2390:0:(events.c:34:request_out_callback()) Process entered 02000000:00000001:2.0:1787829281.639081:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787829281.639085:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829281.639087:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787829281.639088:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:3.0:1787829281.639093:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000008:00100000:2.0:1787829281.639094:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00000001:3.0:1787829281.639098:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000008:00000001:2.0:1787829281.639099:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829281.639102:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829281.639104:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829281.639107:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787829281.639110:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829281.639110:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4298755840 00000100:00000001:2.0:1787829281.639116:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787829281.639117:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787829281.639119:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829281.639123:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 00000100:00000001:2.0:1787829281.639124:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787829281.639132:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2cb40 x1874670646814336/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787829281.639152:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4298755840 00000100:00000001:3.0:1787829281.639163:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 00000100:00000001:2.0:1787829281.639164:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787829281.639169:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829281.639172:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829281.639172:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00080000:2.0:1787829281.639178:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1787829281.639180:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787829281.639197:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1787829281.639198:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00000001:2.0:1787829281.639199:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787829281.639203:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787829281.639205:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787829281.639210:0:468:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4082 00000100:00100000:3.0:1787829281.639211:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829281/real 1787829281] req@ffff8d30c9e2e1c0 x1874670646814080/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829306 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829281.639213:0:468:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787829281.639218:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829281.639221:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787829281.639229:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787829281.639231:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000400:00000001:3.0:1787829281.639232:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000020:00000001:2.0:1787829281.639235:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000400:00000001:3.0:1787829281.639235:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829281.639240:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000020:00000001:2.0:1787829281.639242:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:3.0:1787829281.639244:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787829281.639248:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829281.639248:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829281.639254:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:3.0:1787829281.639254:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829281.639257:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:2.0:1787829281.639266:0:468:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00100000:3.0:1787829281.639267:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2e1c0 x1874670646814080/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829306 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787829281.639276:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829281.639283:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787829281.639287:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829281.639289:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:2.0:1787829281.639292:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829281.639294:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787829281.639295:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1787829281.639301:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00000001:3.0:1787829281.639301:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829281.639305:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000008:00000001:2.0:1787829281.639307:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829281.639308:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829281.639312:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829281.639313:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829281.639316:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 02000000:00000001:2.0:1787829281.639319:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00080000:3.0:1787829281.639320:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 02000000:00000001:2.0:1787829281.639324:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787829281.639327:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:2.0:1787829281.639329:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:3.0:1787829281.639331:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 02000000:00000001:2.0:1787829281.639332:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787829281.639336:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829281.639339:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646814080:192.168.203.143@tcp:250:kworker.0 00000100:00000001:2.0:1787829281.639342:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829281.639345:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829281.639348:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829281.639351:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 00000100:00100000:2.0:1787829281.639352:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2d2c0 x1874670646814464/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:3.0:1787829281.639354:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829281.639358:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829281.639361:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000100:00000001:2.0:1787829281.639365:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000020:00000001:3.0:1787829281.639365:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 00000100:00000001:2.0:1787829281.639369:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829281.639372:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 00000100:00000001:2.0:1787829281.639373:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829281.639375:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829281.639380:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829281.639382:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829281.639384:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829281.639391:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787829281.639392:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829281.639395:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1787829281.639397:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787829281.639399:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787829281.639400:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787829281.639401:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829281.639403:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787829281.639405:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829281.639407:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829281.639411:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2d680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646814208:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829281.639413:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1787829281.639415:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1787829281.639416:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829281.639418:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1787829281.639420:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829281.639425:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787829281.639426:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417874048 : -126234291677568 : ffff8d30c9e2d680) 00000100:00000001:3.0:1787829281.639430:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1787829281.639441:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:3.0:1787829281.639451:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787829281.639453:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829281.639454:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829281.639456:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1787829281.639458:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829281.639461:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298754582 00000100:00000001:3.0:1787829281.639464:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298755840 00000100:00000001:3.0:1787829281.639465:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298754582 00000100:00000001:3.0:1787829281.639467:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298755840 00000100:00000001:3.0:1787829281.639469:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829281.639470:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829281.639471:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1787829281.639475:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829281/real 1787829281] req@ffff8d30c9e2d680 x1874670646814208/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829306 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1787829281.639481:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787829281.639482:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829281.639484:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829281.639485:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829281.639486:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829281.639487:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829281.639489:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829281.639491:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2d680 x1874670646814208/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829306 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829281.639506:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829281.639509:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829281.639512:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829281.639514:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829281.639515:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829281.639518:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829281.639519:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829281.639522:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829281.639527:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829281.639529:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829281.639534:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2d680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646814208:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829281.639539:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787829281.639550:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787829281.639552:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829281.639555:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787829281.639557:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829281.639558:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829281.639561:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646814336:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829281.639564:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1787829281.639565:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1787829281.639567:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829281.639568:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1787829281.639571:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829281.639575:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787829281.639576:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417871168 : -126234291680448 : ffff8d30c9e2cb40) 00000100:00000001:3.0:1787829281.639579:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1787829281.639589:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:3.0:1787829281.639594:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787829281.639595:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829281.639596:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829281.639598:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1787829281.639624:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829281.639628:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298754583 00000100:00000001:3.0:1787829281.639630:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298755840 00000100:00000001:3.0:1787829281.639631:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298754583 00000100:00000001:3.0:1787829281.639633:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298755840 00000100:00000001:3.0:1787829281.639634:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829281.639636:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829281.639637:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1787829281.639640:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829281/real 1787829281] req@ffff8d30c9e2cb40 x1874670646814336/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829306 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1787829281.639646:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787829281.639647:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829281.639649:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829281.639650:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829281.639651:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829281.639653:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829281.639654:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829281.639656:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2cb40 x1874670646814336/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829306 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829281.639673:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829281.639674:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829281.639676:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829281.639677:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829281.639678:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829281.639679:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829281.639680:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829281.639682:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829281.639684:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829281.639686:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829281.639688:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646814336:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829281.639691:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787829281.639692:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787829281.639694:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829281.639695:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787829281.639696:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829281.639698:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829281.639700:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2d2c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646814464:192.168.203.143@tcp:8:kworker.0 00000100:00000001:3.0:1787829281.639702:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1787829281.639704:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1787829281.639705:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829281.639706:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1787829281.639708:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829281.639712:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787829281.639713:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417873088 : -126234291678528 : ffff8d30c9e2d2c0) 00000100:00000001:3.0:1787829281.639716:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1787829281.639724:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:3.0:1787829281.639727:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787829281.639729:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829281.639730:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829281.639731:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1787829281.639733:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829281.639735:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298754596 00000100:00000001:3.0:1787829281.639737:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298755840 00000100:00000001:3.0:1787829281.639739:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298754596 00000100:00000001:3.0:1787829281.639740:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298755840 00000100:00000001:3.0:1787829281.639742:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829281.639743:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829281.639744:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1787829281.639747:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829281/real 1787829281] req@ffff8d30c9e2d2c0 x1874670646814464/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829306 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1787829281.639762:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787829281.639763:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829281.639765:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829281.639766:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829281.639767:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829281.639768:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829281.639770:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829281.639774:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2d2c0 x1874670646814464/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829306 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829281.639782:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829281.639784:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829281.639787:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829281.639788:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829281.639790:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829281.639792:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829281.639794:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829281.639797:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829281.639801:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829281.639804:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829281.639808:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2d2c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646814464:192.168.203.143@tcp:8:kworker.0 00000100:00000001:3.0:1787829281.639823:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829281.639825:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829281.639827:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829281.639829:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829281.639831:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829281.639833:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829281.639835:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829281.639837:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829281.639838:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829281.639842:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829281.639843:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829281.639845:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829281.639846:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829281.639847:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829281.639848:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829281.639849:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829281.639850:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829281.639851:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829281.639852:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829281.639853:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829281.639854:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829281.639855:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829281.639856:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829281.639857:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829281.639858:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829281.639860:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829281.639861:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829281.639863:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829281.639864:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829281.639866:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829281.639867:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829281.639869:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829281.639873:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787829281.639874:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829281.639877:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1787829281.639879:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829282.534713:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829282.534720:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829282.534768:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829282.534772:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829282.534774:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829283.558670:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829283.558676:0:815:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1787829283.558678:0:815:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1787829283.558681:0:815:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829283.558686:0:815:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1787829283.558688:0:815:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829283.558690:0:815:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829283.558691:0:815:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829283.558747:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829283.558750:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829283.558753:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829284.582674:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829284.582681:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829284.582751:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829284.582773:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829284.582777:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829285.606718:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829285.606725:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829285.606779:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829285.606783:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829285.606785:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829286.630682:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829286.630687:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829286.630737:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829286.630742:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829286.630747:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787829286.758807:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829286.758838:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1787829286.758841:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1787829286.758846:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787829286.758852:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1787829286.758862:0:468:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 4087 00000100:00080000:1.0:1787829286.758893:0:468:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1787829286.758898:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787829286.758902:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:1.0:1787829286.758923:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:1.0:1787829286.758927:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:1.0:1787829286.758930:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:1.0:1787829286.758933:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:1.0:1787829286.758936:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787829286.758937:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:1.0:1787829286.758951:0:468:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:1.0:1787829286.758955:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829286.758957:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1787829286.758961:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829286.758964:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829286.758967:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:1.0:1787829286.758970:0:468:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829286.758990:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1787829286.758992:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829286.758996:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1787829286.758998:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1787829286.759011:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829286.759023:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829286.759041:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d13ee940 x1874670646814592/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787829286.759053:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787829286.759057:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829286.759079:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1787829286.759082:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829286.759086:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1787829286.759088:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1787829286.759098:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787829286.759102:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:3.0:1787829286.759135:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1787829286.759142:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00080000:1.0:1787829286.759144:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4087 00000100:00000001:3.0:1787829286.759147:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00080000:1.0:1787829286.759152:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787829286.759153:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787829286.759176:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829286.759182:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787829286.759186:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829286.759188:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829286.759195:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d13ee940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646814592:192.168.203.143@tcp:250:kworker.0 00000100:00000001:3.0:1787829286.759201:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:1.0:1787829286.759204:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 02000000:00000001:3.0:1787829286.759208:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:1.0:1787829286.759209:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 02000000:00000001:3.0:1787829286.759214:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829286.759215:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:1.0:1787829286.759219:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 02000000:00000001:3.0:1787829286.759222:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000020:00000001:1.0:1787829286.759224:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 02000000:00000001:3.0:1787829286.759247:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829286.759254:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:1.0:1787829286.759257:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787829286.759259:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:1.0:1787829286.759264:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:1.0:1787829286.759267:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829286.759269:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:3.0:1787829286.759270:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787829286.759276:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839541348672 : -126234168202944 : ffff8d30d13ee940) 00000100:00000001:3.0:1787829286.759282:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 02000000:00000001:1.0:1787829286.759286:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829286.759291:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829286.759293:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:1.0:1787829286.759298:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1787829286.759301:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829286.759303:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829286.759320:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1787829286.759322:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829286.759324:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1787829286.759325:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1787829286.759327:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:3.0:1787829286.759335:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787829286.759344:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829286.759353:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00100000:1.0:1787829286.759354:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d13ec3c0 x1874670646814720/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787829286.759360:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829286.759365:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829286.759367:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1787829286.759373:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829286.759381:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787829286.759383:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4298760960 00000100:00000001:1.0:1787829286.759386:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829286.759397:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 00000100:00000001:1.0:1787829286.759402:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1787829286.759408:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1787829286.759408:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4298760960 00000100:00080000:1.0:1787829286.759418:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:1.0:1787829286.759420:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:3.0:1787829286.759421:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 00000100:00080000:1.0:1787829286.759422:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787829286.759430:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829286.759438:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829286.759439:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1787829286.759446:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4087 00000100:00000001:3.0:1787829286.759447:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00080000:1.0:1787829286.759451:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1787829286.759461:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829286/real 1787829286] req@ffff8d30d13ee940 x1874670646814592/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829311 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787829286.759463:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787829286.759466:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:1.0:1787829286.759472:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:1.0:1787829286.759476:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000400:00000001:3.0:1787829286.759490:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000020:00000001:1.0:1787829286.759492:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000400:00000001:3.0:1787829286.759494:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000020:00000001:1.0:1787829286.759500:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:3.0:1787829286.759503:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787829286.759504:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787829286.759509:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:3.0:1787829286.759511:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829286.759516:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:1.0:1787829286.759517:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:1.0:1787829286.759524:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829286.759524:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 02000000:00000001:1.0:1787829286.759530:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:3.0:1787829286.759530:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:1.0:1787829286.759536:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829286.759542:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829286.759544:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d13ee940 x1874670646814592/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829311 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787829286.759546:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:1.0:1787829286.759550:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00000001:3.0:1787829286.759564:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000008:00000001:1.0:1787829286.759566:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829286.759572:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829286.759575:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 02000000:00000001:1.0:1787829286.759577:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1787829286.759580:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829286.759585:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:3.0:1787829286.759587:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:1.0:1787829286.759588:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:3.0:1787829286.759594:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829286.759597:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829286.759601:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829286.759604:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829286.759609:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829286.759666:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 02000000:00000001:1.0:1787829286.759670:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829286.759672:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000001:1.0:1787829286.759680:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829286.759682:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d13ee940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646814592:192.168.203.143@tcp:250:kworker.0 00000100:00100000:1.0:1787829286.759692:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d13ec780 x1874670646814848/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787829286.759693:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829286.759701:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829286.759705:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829286.759710:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 00000100:00000001:1.0:1787829286.759712:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 02000000:00000001:3.0:1787829286.759716:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00000001:1.0:1787829286.759718:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829286.759721:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829286.759726:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 00000100:00000001:1.0:1787829286.759727:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829286.759733:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 00000100:00080000:1.0:1787829286.759734:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 02000000:00000001:3.0:1787829286.759739:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00080000:1.0:1787829286.759743:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1787829286.759747:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787829286.759748:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:3.0:1787829286.759755:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:1.0:1787829286.759756:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787829286.759765:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829286.759766:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1787829286.759775:0:468:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4087 00000100:00000001:3.0:1787829286.759781:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00080000:1.0:1787829286.759783:0:468:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787829286.759787:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829286.759790:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787829286.759795:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:3.0:1787829286.759796:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:1.0:1787829286.759799:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:3.0:1787829286.759803:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787829286.759806:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000020:00000001:1.0:1787829286.759810:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:3.0:1787829286.759812:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000020:00000001:1.0:1787829286.759815:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:3.0:1787829286.759816:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829286.759823:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 02000000:00000001:3.0:1787829286.759825:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:1.0:1787829286.759829:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 02000000:00000001:3.0:1787829286.759831:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829286.759833:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 02000000:00000001:3.0:1787829286.759836:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1787829286.759844:0:468:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00100000:3.0:1787829286.759846:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d13ec3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646814720:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787829286.759855:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829286.759856:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1787829286.759861:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787829286.759863:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787829286.759864:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829286.759869:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787829286.759871:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829286.759873:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829286.759882:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829286.759886:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787829286.759887:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:1.0:1787829286.759892:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000008:00000001:1.0:1787829286.759896:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829286.759901:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839541339072 : -126234168212544 : ffff8d30d13ec3c0) 00000100:00000001:1.0:1787829286.759903:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829286.759911:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 02000000:00000001:1.0:1787829286.759916:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1787829286.759919:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829286.759922:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1787829286.759924:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1787829286.759927:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829286.759935:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:3.0:1787829286.759938:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00100000:1.0:1787829286.759945:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d13ec000 x1874670646814976/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787829286.759947:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787829286.759952:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829286.759956:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829286.759962:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787829286.759964:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787829286.759968:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829286.759975:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829286.759976:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829286.759989:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298759703 00000100:00000001:3.0:1787829286.759995:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298760960 00000100:00000001:3.0:1787829286.759999:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298759703 00000100:00000001:3.0:1787829286.760004:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298760960 00000100:00000001:3.0:1787829286.760008:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829286.760012:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829286.760015:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1787829286.760025:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829286/real 1787829286] req@ffff8d30d13ec3c0 x1874670646814720/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829311 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1787829286.760040:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787829286.760042:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829286.760046:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829286.760048:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829286.760049:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829286.760051:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829286.760053:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829286.760058:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d13ec3c0 x1874670646814720/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829311 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829286.760065:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829286.760067:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829286.760070:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829286.760071:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829286.760073:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829286.760074:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829286.760076:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829286.760079:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829286.760082:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829286.760084:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829286.760087:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d13ec3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646814720:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829286.760090:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787829286.760092:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787829286.760093:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829286.760095:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787829286.760097:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829286.760099:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829286.760102:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d13ec780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646814848:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829286.760125:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1787829286.760127:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1787829286.760129:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829286.760131:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1787829286.760135:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829286.760141:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787829286.760142:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839541340032 : -126234168211584 : ffff8d30d13ec780) 00000100:00000001:3.0:1787829286.760146:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1787829286.760162:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:3.0:1787829286.760167:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787829286.760170:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829286.760172:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829286.760173:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1787829286.760176:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829286.760180:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298759703 00000100:00000001:3.0:1787829286.760182:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298760960 00000100:00000001:3.0:1787829286.760184:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298759703 00000100:00000001:3.0:1787829286.760186:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298760960 00000100:00000001:3.0:1787829286.760188:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829286.760190:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829286.760192:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1787829286.760196:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829286/real 1787829286] req@ffff8d30d13ec780 x1874670646814848/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829311 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1787829286.760203:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787829286.760204:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829286.760207:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829286.760208:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829286.760209:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829286.760211:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829286.760213:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829286.760216:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d13ec780 x1874670646814848/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829311 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829286.760222:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829286.760224:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829286.760226:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829286.760228:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829286.760229:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829286.760230:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829286.760232:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829286.760235:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829286.760238:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829286.760240:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829286.760244:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d13ec780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646814848:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829286.760249:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829286.760252:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829286.760254:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829286.760256:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829286.760259:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829286.760262:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829286.760264:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829286.760268:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829286.760270:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829286.760273:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829286.760275:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829286.760278:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829286.760279:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829286.760281:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829286.760284:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829286.760286:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829286.760287:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829286.760290:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829286.760292:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829286.760294:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829286.760296:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829286.760299:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829286.760304:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787829286.760306:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829286.760310:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1787829286.760312:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787829286.760315:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787829286.760317:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787829286.760319:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829286.760323:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787829286.760325:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829286.760328:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829286.760333:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d13ec000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646814976:192.168.203.143@tcp:8:kworker.0 00000100:00000001:3.0:1787829286.760339:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1787829286.760342:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1787829286.760344:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829286.760346:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1787829286.760352:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829286.760360:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787829286.760362:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839541338112 : -126234168213504 : ffff8d30d13ec000) 00000100:00000001:3.0:1787829286.760367:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1787829286.760388:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:3.0:1787829286.760396:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787829286.760400:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829286.760402:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829286.760405:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1787829286.760408:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829286.760412:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298759716 00000100:00000001:3.0:1787829286.760415:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298760960 00000100:00000001:3.0:1787829286.760417:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298759716 00000100:00000001:3.0:1787829286.760419:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298760960 00000100:00000001:3.0:1787829286.760421:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829286.760423:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829286.760425:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1787829286.760430:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829286/real 1787829286] req@ffff8d30d13ec000 x1874670646814976/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829311 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1787829286.760437:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787829286.760438:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829286.760441:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829286.760442:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829286.760444:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829286.760446:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829286.760447:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829286.760450:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d13ec000 x1874670646814976/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829311 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829286.760457:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829286.760460:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829286.760462:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829286.760463:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829286.760465:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829286.760467:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829286.760469:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829286.760471:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829286.760474:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829286.760476:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829286.760479:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d13ec000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646814976:192.168.203.143@tcp:8:kworker.0 00000100:00000001:3.0:1787829286.760482:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829286.760484:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829286.760485:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829286.760487:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829286.760489:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829286.760490:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829286.760492:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829286.760494:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829286.760495:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829286.760497:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829286.760498:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829286.760500:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829286.760504:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787829286.760505:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829286.760507:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1787829286.760509:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829287.654717:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829287.654724:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829287.654781:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829287.654784:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829287.654787:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829288.678758:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829288.678765:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829288.678815:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829288.678830:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829288.678833:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829289.702722:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829289.702727:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829289.702800:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829289.702807:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829289.702813:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829290.726703:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829290.726708:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829290.726739:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829290.726743:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829290.726746:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829291.750684:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829291.750690:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829291.750766:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829291.750771:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829291.750775:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787829291.878730:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829291.878744:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1787829291.878748:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787829291.878752:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787829291.878778:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787829291.878784:0:468:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 4092 00000100:00080000:2.0:1787829291.878797:0:468:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787829291.878800:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829291.878803:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787829291.878807:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787829291.878809:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787829291.878812:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:2.0:1787829291.878814:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:2.0:1787829291.878816:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829291.878817:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:2.0:1787829291.878820:0:468:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787829291.878823:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829291.878825:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787829291.878828:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787829291.878830:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829291.878831:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:2.0:1787829291.878833:0:468:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829291.878841:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787829291.878843:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829291.878845:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787829291.878847:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787829291.878850:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829291.878863:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787829291.878872:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2fc00 x1874670646815104/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787829291.878880:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787829291.878884:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829291.878897:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787829291.878900:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829291.878904:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1787829291.878906:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787829291.878907:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787829291.878910:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787829291.878912:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4092 00000100:00080000:2.0:1787829291.878915:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787829291.878917:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829291.878918:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787829291.878920:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787829291.878921:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787829291.878923:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000020:00000001:2.0:1787829291.878925:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:2.0:1787829291.878927:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829291.878928:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:2.0:1787829291.878930:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787829291.878931:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829291.878933:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1787829291.878934:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787829291.878938:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 02000000:00000001:2.0:1787829291.878938:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829291.878942:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 02000000:00000001:2.0:1787829291.878943:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829291.878946:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:1.0:1787829291.878947:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787829291.878949:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000008:00100000:2.0:1787829291.878951:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 02000000:00000001:1.0:1787829291.878954:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000008:00000001:2.0:1787829291.878954:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829291.878958:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787829291.878958:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829291.878964:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829291.878965:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787829291.878968:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829291.878969:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646815104:192.168.203.143@tcp:250:kworker.0 00000100:00000001:2.0:1787829291.878971:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787829291.878973:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:1.0:1787829291.878976:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787829291.878976:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829291.878980:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:2.0:1787829291.878982:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829291.878983:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829291.878987:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00100000:2.0:1787829291.878988:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2c3c0 x1874670646815232/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:1.0:1787829291.878991:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829291.879003:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787829291.879004:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829291.879006:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787829291.879008:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829291.879009:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417883648 : -126234291667968 : ffff8d30c9e2fc00) 00000100:00080000:2.0:1787829291.879011:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1787829291.879013:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00080000:2.0:1787829291.879014:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1787829291.879017:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787829291.879019:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787829291.879020:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787829291.879022:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4092 00000100:00080000:2.0:1787829291.879026:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787829291.879027:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829291.879028:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787829291.879030:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787829291.879031:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787829291.879032:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000020:00000001:2.0:1787829291.879034:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:2.0:1787829291.879036:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829291.879037:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:2.0:1787829291.879039:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787829291.879040:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829291.879041:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787829291.879043:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787829291.879045:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829291.879047:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1787829291.879048:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1787829291.879049:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829291.879051:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829291.879056:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4298766080 00000100:00000001:2.0:1787829291.879056:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829291.879062:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787829291.879064:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829291.879065:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 00000100:00000001:2.0:1787829291.879066:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787829291.879078:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787829291.879081:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829291.879085:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4298766080 00000100:00000001:2.0:1787829291.879086:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829291.879092:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 00000100:00100000:2.0:1787829291.879093:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2de00 x1874670646815360/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787829291.879096:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829291.879102:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829291.879117:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829291.879117:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787829291.879120:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829291.879121:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829291.879123:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787829291.879126:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829291.879128:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1787829291.879129:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1787829291.879129:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:1.0:1787829291.879133:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00080000:2.0:1787829291.879133:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787829291.879138:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787829291.879139:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:1.0:1787829291.879141:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00080000:2.0:1787829291.879143:0:468:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4092 00000100:00000001:1.0:1787829291.879144:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787829291.879147:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787829291.879149:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787829291.879150:0:468:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 02000000:00000001:1.0:1787829291.879153:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:2.0:1787829291.879154:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 02000000:00000001:1.0:1787829291.879157:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787829291.879157:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787829291.879163:0:468:0:(genops.c:792:class_conn2export()) Process entered 02000000:00000001:1.0:1787829291.879164:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1787829291.879166:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787829291.879170:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00100000:1.0:1787829291.879171:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646815232:192.168.203.143@tcp:38:kworker.0 00000020:00000001:2.0:1787829291.879176:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:1.0:1787829291.879177:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:2.0:1787829291.879179:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 02000000:00000001:1.0:1787829291.879181:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:2.0:1787829291.879182:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 02000000:00000001:1.0:1787829291.879184:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787829291.879187:0:468:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 02000000:00000001:1.0:1787829291.879188:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:2.0:1787829291.879192:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829291.879193:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829291.879195:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787829291.879199:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787829291.879208:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829291.879209:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:1.0:1787829291.879211:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000008:00100000:2.0:1787829291.879212:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00000001:1.0:1787829291.879214:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417869248 : -126234291682368 : ffff8d30c9e2c3c0) 00000008:00000001:2.0:1787829291.879216:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829291.879220:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:2.0:1787829291.879220:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829291.879228:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787829291.879230:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829291.879233:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787829291.879235:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:1.0:1787829291.879236:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829291.879237:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829291.879241:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298764822 00000100:00000001:1.0:1787829291.879244:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298766080 00000100:00000001:2.0:1787829291.879246:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829291.879247:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298764822 00000100:00000001:1.0:1787829291.879252:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298766080 00000100:00000001:1.0:1787829291.879254:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829291.879257:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787829291.879258:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2c780 x1874670646815488/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787829291.879260:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787829291.879263:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787829291.879265:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829291.879269:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787829291.879272:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787829291.879275:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 02000000:00000001:1.0:1787829291.879276:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829291.879280:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829291.879282:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646815360:192.168.203.143@tcp:38:kworker.0 00000100:00000001:2.0:1787829291.879285:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829291.879286:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787829291.879289:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787829291.879290:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829291.879292:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787829291.879295:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829291.879299:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787829291.879301:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417875968 : -126234291675648 : ffff8d30c9e2de00) 00000100:00000001:1.0:1787829291.879303:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:1.0:1787829291.879313:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829291.879315:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298764822 00000100:00000001:1.0:1787829291.879318:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298766080 00000100:00000001:1.0:1787829291.879320:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298764822 00000100:00000001:1.0:1787829291.879322:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298766080 00000100:00000001:1.0:1787829291.879323:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829291.879325:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829291.879327:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829291.879328:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829291.879330:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787829291.879332:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:1.0:1787829291.879333:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787829291.879334:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1787829291.879335:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787829291.879336:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787829291.879336:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829291.879338:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787829291.879339:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829291.879340:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829291.879342:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646815488:192.168.203.143@tcp:8:kworker.0 00000100:00000001:1.0:1787829291.879344:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787829291.879345:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787829291.879346:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829291.879347:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787829291.879348:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829291.879351:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787829291.879352:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417870208 : -126234291681408 : ffff8d30c9e2c780) 00000100:00000001:1.0:1787829291.879354:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:1.0:1787829291.879359:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829291.879361:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298764835 00000100:00000001:1.0:1787829291.879363:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298766080 00000100:00000001:1.0:1787829291.879364:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298764835 00000100:00000001:1.0:1787829291.879365:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298766080 00000100:00000001:1.0:1787829291.879367:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829291.879368:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829291.879369:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829291.879370:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829291.879371:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787829291.879372:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:1.0:1787829291.879373:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787829291.879374:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1787829291.879375:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829291.879376:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829292.774718:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829292.774723:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829292.774802:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829292.774809:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829292.774814:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829293.798703:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829293.798709:0:815:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1787829293.798711:0:815:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1787829293.798714:0:815:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829293.798719:0:815:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1787829293.798721:0:815:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829293.798722:0:815:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829293.798725:0:815:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829293.798778:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829293.798783:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829293.798786:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829294.822806:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829294.822817:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829294.822908:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829294.822916:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829294.822922:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:3.0:1787829294.950983:0:2378:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.143:988 00000400:00000100:3.0:1787829294.951042:0:2378:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.143@tcp at host 192.168.203.143 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:3.0:1787829294.951082:0:2378:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.143@tcp 00000100:00000001:3.0:1787829294.951102:0:2378:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787829294.951133:0:2378:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829294.951137:0:2378:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829294.951141:0:2378:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:3.0:1787829294.951157:0:2378:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.143@tcp 00000100:00000001:3.0:1787829294.951164:0:2378:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787829294.951168:0:2378:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829294.951170:0:2378:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829294.951172:0:2378:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:3.0:1787829294.951179:0:2378:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.143@tcp 00000100:00000001:3.0:1787829294.951185:0:2378:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787829294.951188:0:2378:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829294.951190:0:2378:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829294.951192:0:2378:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:3.0:1787829294.951216:0:2378:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.143@tcp 00000100:00000001:3.0:1787829294.951221:0:2378:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787829294.951224:0:2378:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829294.951225:0:2378:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829294.951227:0:2378:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787829294.951724:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787829294.951732:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1787829294.951737:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787829294.951751:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829291/real 1787829294] req@ffff8d30c9e2c780 x1874670646815488/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829316 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787829294.951770:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787829294.951772:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787829294.951781:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787829294.951785:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787829294.951787:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829294.951791:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829294.951795:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787829294.951801:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2c780 x1874670646815488/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829316 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787829294.951811:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829294.951815:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787829294.951819:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829294.951821:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829294.951823:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829294.951827:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787829294.951830:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787829294.951835:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829294.951842:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787829294.951847:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787829294.951853:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646815488:192.168.203.143@tcp:8:kworker.0 00000100:00000001:1.0:1787829294.951860:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787829294.951865:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829291/real 1787829294] req@ffff8d30c9e2c3c0 x1874670646815232/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829316 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787829294.951873:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787829294.951875:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787829294.951878:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787829294.951879:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787829294.951881:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829294.951883:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829294.951884:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787829294.951887:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2c3c0 x1874670646815232/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829316 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787829294.951892:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829294.951893:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787829294.951895:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829294.951896:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829294.951896:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829294.951898:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787829294.951900:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787829294.951903:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829294.951907:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787829294.951909:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787829294.951911:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646815232:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787829294.951914:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787829294.951918:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829291/real 1787829294] req@ffff8d30c9e2de00 x1874670646815360/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829316 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787829294.951923:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787829294.951924:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787829294.951927:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787829294.951928:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787829294.951930:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829294.951931:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829294.951932:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787829294.951935:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2de00 x1874670646815360/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829316 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787829294.951942:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829294.951944:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787829294.951948:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829294.951950:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829294.951952:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829294.951954:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787829294.951956:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787829294.951959:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829294.951965:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787829294.951968:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787829294.951973:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646815360:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787829294.951979:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787829294.951984:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829291/real 1787829294] req@ffff8d30c9e2fc00 x1874670646815104/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829316 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787829294.951994:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787829294.951996:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787829294.952001:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787829294.952003:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787829294.952004:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829294.952006:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829294.952007:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787829294.952011:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2fc00 x1874670646815104/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829316 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787829294.952016:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829294.952018:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787829294.952020:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829294.952021:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829294.952022:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829294.952023:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787829294.952024:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787829294.952026:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829294.952028:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787829294.952031:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787829294.952033:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646815104:192.168.203.143@tcp:250:kworker.0 00000100:00000001:1.0:1787829294.952037:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829294.952039:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829294.952041:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787829294.952044:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787829294.952072:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787829294.952075:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787829294.952077:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787829294.952082:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787829294.952085:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787829294.952089:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787829294.952091:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829294.952094:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829294.952095:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787829294.952097:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787829294.952100:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787829294.952101:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787829294.952103:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787829294.952105:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787829294.952106:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787829294.952108:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787829294.952109:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829294.952111:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829294.952113:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787829294.952114:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787829294.952117:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787829294.952119:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787829294.952120:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787829294.952122:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787829294.952123:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787829294.952130:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787829294.952132:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829294.952134:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829294.952135:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787829294.952137:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787829294.952138:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787829294.952140:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787829294.952141:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787829294.952143:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787829294.952145:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787829294.952148:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787829294.952149:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829294.952152:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829294.952161:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787829294.952164:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829294.952167:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787829294.952170:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829295.846745:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829295.846751:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829295.846812:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829295.846817:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829295.846820:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829296.870697:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829296.870704:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829296.870788:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829296.870792:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829296.870795:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787829296.998765:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829296.998778:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1787829296.998781:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787829296.998786:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787829296.998792:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787829296.998803:0:468:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 4097 00000100:00080000:2.0:1787829296.998817:0:468:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787829296.998822:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829296.998824:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787829296.998829:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787829296.998839:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787829296.998843:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:2.0:1787829296.998845:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:2.0:1787829296.998848:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829296.998849:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:2.0:1787829296.998853:0:468:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787829296.998856:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829296.998858:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787829296.998862:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787829296.998864:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829296.998867:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:2.0:1787829296.998870:0:468:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829296.998880:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787829296.998882:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829296.998886:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787829296.998890:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787829296.998896:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829296.998927:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787829296.998940:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2f0c0 x1874670646815616/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787829296.998954:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787829296.998960:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829296.998984:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787829296.998989:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829296.998992:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1787829296.998994:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787829296.998997:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787829296.999000:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787829296.999005:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4097 00000100:00080000:2.0:1787829296.999012:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1787829296.999020:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787829296.999026:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1787829296.999027:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787829296.999030:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1787829296.999073:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:1.0:1787829296.999076:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000020:00000001:2.0:1787829296.999079:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:1.0:1787829296.999081:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1787829296.999083:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 02000000:00000001:1.0:1787829296.999090:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000020:00000001:2.0:1787829296.999092:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 02000000:00000001:1.0:1787829296.999096:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000020:00000001:2.0:1787829296.999098:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 02000000:00000001:1.0:1787829296.999103:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829296.999105:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829296.999108:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00100000:1.0:1787829296.999114:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646815616:192.168.203.143@tcp:250:kworker.0 00000100:00080000:2.0:1787829296.999117:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787829296.999125:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829296.999126:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787829296.999128:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787829296.999139:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787829296.999142:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829296.999145:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:2.0:1787829296.999145:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 02000000:00000001:1.0:1787829296.999152:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000008:00100000:2.0:1787829296.999154:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1787829296.999160:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829296.999162:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:2.0:1787829296.999163:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829296.999188:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829296.999203:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787829296.999207:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829296.999209:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787829296.999211:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787829296.999214:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829296.999216:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787829296.999220:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417880768 : -126234291670848 : ffff8d30c9e2f0c0) 00000100:00000001:2.0:1787829296.999222:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829296.999225:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00100000:2.0:1787829296.999232:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2ed00 x1874670646815744/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787829296.999242:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787829296.999245:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829296.999249:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787829296.999252:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829296.999258:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1787829296.999260:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787829296.999262:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787829296.999265:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787829296.999269:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4097 00000100:00080000:2.0:1787829296.999275:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787829296.999279:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829296.999281:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787829296.999285:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787829296.999287:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787829296.999289:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000020:00000001:2.0:1787829296.999292:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:2.0:1787829296.999296:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829296.999298:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000800:00000100:1.0:1787829296.999304:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00080000:2.0:1787829296.999318:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787829296.999322:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829296.999325:0:2390:0:(events.c:34:request_out_callback()) Process entered 02000000:00000001:2.0:1787829296.999325:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787829296.999331:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829296.999335:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 02000000:00000001:2.0:1787829296.999336:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829296.999341:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:1.0:1787829296.999342:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829296.999346:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000008:00100000:2.0:1787829296.999351:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1787829296.999355:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829296.999357:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829296.999359:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829296.999363:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787829296.999366:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829296.999368:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4298771200 00000100:00000001:2.0:1787829296.999369:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787829296.999373:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787829296.999376:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829296.999383:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829296.999388:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 00000100:00100000:2.0:1787829296.999390:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2e940 x1874670646815872/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787829296.999398:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4298771200 00000100:00000001:2.0:1787829296.999403:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787829296.999407:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829296.999408:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 00000100:00000001:2.0:1787829296.999414:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829296.999416:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00080000:2.0:1787829296.999420:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1787829296.999424:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787829296.999426:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1787829296.999430:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:1.0:1787829296.999431:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00080000:2.0:1787829296.999433:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787829296.999439:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00100000:1.0:1787829296.999442:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829296/real 1787829296] req@ffff8d30c9e2f0c0 x1874670646815616/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829321 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829296.999446:0:468:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4097 00000100:00080000:2.0:1787829296.999454:0:468:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787829296.999459:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829296.999462:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787829296.999466:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000400:00000001:1.0:1787829296.999467:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000020:00000001:2.0:1787829296.999468:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000400:00000001:1.0:1787829296.999472:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000020:00000001:2.0:1787829296.999473:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000020:00000001:2.0:1787829296.999478:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:1.0:1787829296.999481:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1787829296.999482:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829296.999487:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:1.0:1787829296.999488:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787829296.999493:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:2.0:1787829296.999493:0:468:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:1.0:1787829296.999500:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787829296.999500:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829296.999503:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1787829296.999506:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:2.0:1787829296.999509:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787829296.999514:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829296.999516:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2f0c0 x1874670646815616/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829321 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787829296.999518:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1787829296.999522:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000008:00000001:2.0:1787829296.999529:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829296.999533:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829296.999540:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:2.0:1787829296.999542:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787829296.999546:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829296.999547:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:1.0:1787829296.999549:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 02000000:00000001:2.0:1787829296.999549:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787829296.999553:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829296.999557:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829296.999561:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787829296.999561:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829296.999563:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829296.999569:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00100000:2.0:1787829296.999572:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2c000 x1874670646816000/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787829296.999574:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787829296.999581:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829296.999593:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:2.0:1787829296.999594:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787829296.999598:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000001:2.0:1787829296.999599:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829296.999606:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646815616:192.168.203.143@tcp:250:kworker.0 00000100:00000001:2.0:1787829296.999638:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829296.999666:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829296.999671:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829296.999673:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787829296.999678:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787829296.999685:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787829296.999688:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787829296.999692:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787829296.999702:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787829296.999708:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787829296.999713:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787829296.999715:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829296.999718:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829296.999734:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787829296.999738:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829296.999741:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787829296.999742:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1787829296.999744:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787829296.999746:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787829296.999748:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829296.999751:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787829296.999753:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829296.999756:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829296.999760:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646815744:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787829296.999764:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787829296.999767:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787829296.999768:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829296.999770:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787829296.999774:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829296.999781:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787829296.999783:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417879808 : -126234291671808 : ffff8d30c9e2ed00) 00000100:00000001:1.0:1787829296.999790:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1787829296.999811:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787829296.999820:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787829296.999823:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829296.999826:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829296.999828:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787829296.999831:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829296.999837:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298769943 00000100:00000001:1.0:1787829296.999841:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298771200 00000100:00000001:1.0:1787829296.999844:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298769943 00000100:00000001:1.0:1787829296.999846:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298771200 00000100:00000001:1.0:1787829296.999850:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829296.999852:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829296.999855:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787829296.999861:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829296/real 1787829296] req@ffff8d30c9e2ed00 x1874670646815744/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829321 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787829296.999870:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787829296.999872:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787829296.999875:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787829296.999878:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787829296.999879:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829296.999881:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829296.999883:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787829296.999887:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2ed00 x1874670646815744/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829321 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787829296.999896:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829296.999899:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787829296.999901:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829296.999903:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829296.999904:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829296.999906:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787829296.999908:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787829296.999911:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829296.999915:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787829296.999917:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787829296.999920:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646815744:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787829296.999925:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787829296.999926:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787829296.999928:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829296.999930:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787829296.999932:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829296.999934:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829296.999937:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646815872:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787829296.999943:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787829296.999945:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787829296.999947:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829296.999948:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787829296.999952:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829296.999957:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787829296.999959:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417878848 : -126234291672768 : ffff8d30c9e2e940) 00000100:00000001:1.0:1787829296.999963:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1787829296.999974:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787829296.999978:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787829296.999981:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829296.999982:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829296.999984:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787829296.999986:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829296.999990:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298769943 00000100:00000001:1.0:1787829296.999993:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298771200 00000100:00000001:1.0:1787829296.999995:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298769943 00000100:00000001:1.0:1787829296.999997:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298771200 00000100:00000001:1.0:1787829296.999999:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829297.000001:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829297.000003:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787829297.000007:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829296/real 1787829296] req@ffff8d30c9e2e940 x1874670646815872/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829321 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787829297.000015:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787829297.000016:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787829297.000019:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787829297.000021:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787829297.000023:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829297.000024:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829297.000026:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787829297.000029:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2e940 x1874670646815872/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829321 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787829297.000060:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829297.000063:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787829297.000065:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829297.000067:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829297.000068:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829297.000082:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787829297.000083:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787829297.000086:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829297.000089:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787829297.000091:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787829297.000094:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646815872:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787829297.000098:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787829297.000100:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787829297.000101:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829297.000103:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787829297.000105:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829297.000107:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829297.000110:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646816000:192.168.203.143@tcp:8:kworker.0 00000100:00000001:1.0:1787829297.000113:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787829297.000116:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787829297.000117:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829297.000119:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787829297.000122:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829297.000129:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787829297.000131:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417868288 : -126234291683328 : ffff8d30c9e2c000) 00000100:00000001:1.0:1787829297.000134:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1787829297.000157:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787829297.000161:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787829297.000164:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829297.000165:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829297.000167:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787829297.000170:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829297.000173:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298769956 00000100:00000001:1.0:1787829297.000175:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298771200 00000100:00000001:1.0:1787829297.000177:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298769956 00000100:00000001:1.0:1787829297.000180:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298771200 00000100:00000001:1.0:1787829297.000182:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829297.000184:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829297.000186:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787829297.000190:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829296/real 1787829297] req@ffff8d30c9e2c000 x1874670646816000/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829321 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787829297.000199:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787829297.000201:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787829297.000203:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787829297.000205:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787829297.000207:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829297.000208:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829297.000210:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787829297.000214:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2c000 x1874670646816000/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829321 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787829297.000220:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829297.000223:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787829297.000226:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829297.000227:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829297.000228:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829297.000230:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787829297.000232:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787829297.000235:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829297.000238:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787829297.000240:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787829297.000244:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646816000:192.168.203.143@tcp:8:kworker.0 00000100:00000001:1.0:1787829297.000248:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829297.000250:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829297.000251:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787829297.000253:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787829297.000255:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787829297.000257:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787829297.000258:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787829297.000261:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787829297.000263:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787829297.000269:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787829297.000270:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829297.000272:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829297.000273:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787829297.000275:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787829297.000278:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787829297.000279:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787829297.000280:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787829297.000282:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787829297.000283:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787829297.000284:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787829297.000286:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829297.000287:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829297.000289:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787829297.000290:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787829297.000291:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787829297.000293:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787829297.000294:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787829297.000295:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787829297.000296:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787829297.000299:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787829297.000301:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829297.000303:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829297.000308:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787829297.000309:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829297.000312:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787829297.000314:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829297.894746:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829297.894756:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829297.894823:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829297.894827:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829297.894830:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829298.918743:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829298.918752:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829298.918833:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829298.918839:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829298.918844:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829299.942809:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829299.942817:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829299.942910:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829299.942916:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829299.942920:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829300.966763:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829300.966774:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829300.966856:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829300.966861:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829300.966864:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829301.990714:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829301.990722:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829301.990799:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829301.990803:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829301.990806:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787829302.118757:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829302.118769:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1787829302.118772:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1787829302.118777:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787829302.118781:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1787829302.118788:0:468:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 4102 00000100:00080000:1.0:1787829302.118800:0:468:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1787829302.118803:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787829302.118804:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:1.0:1787829302.118809:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:1.0:1787829302.118811:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:1.0:1787829302.118815:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:1.0:1787829302.118817:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:1.0:1787829302.118819:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787829302.118820:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:1.0:1787829302.118824:0:468:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:1.0:1787829302.118826:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829302.118828:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1787829302.118832:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829302.118834:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829302.118836:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:1.0:1787829302.118838:0:468:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829302.118851:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1787829302.118852:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829302.118855:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1787829302.118857:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1787829302.118861:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829302.118874:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829302.118883:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d13ecb40 x1874670646816128/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787829302.118892:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787829302.118896:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829302.118912:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1787829302.118915:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829302.118919:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1787829302.118920:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1787829302.118922:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787829302.118924:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1787829302.118926:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4102 00000100:00080000:1.0:1787829302.118938:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1787829302.118939:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787829302.118940:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:1.0:1787829302.118941:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:1.0:1787829302.118942:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:1.0:1787829302.118944:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000020:00000001:1.0:1787829302.118945:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:1.0:1787829302.118947:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787829302.118949:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:1.0:1787829302.118951:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:1.0:1787829302.118952:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829302.118954:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1787829302.118960:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829302.118962:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829302.118963:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:1.0:1787829302.118968:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00000001:3.0:1787829302.118970:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000008:00000001:1.0:1787829302.118971:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829302.118975:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829302.118977:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 02000000:00000001:1.0:1787829302.118978:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1787829302.118981:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829302.118983:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:3.0:1787829302.118983:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 02000000:00000001:1.0:1787829302.118987:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:3.0:1787829302.118988:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 02000000:00000001:1.0:1787829302.118989:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829302.118991:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829302.118995:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829302.119010:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000100:00100000:1.0:1787829302.119015:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d13ef0c0 x1874670646816256/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:3.0:1787829302.119017:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829302.119022:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829302.119030:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d13ecb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646816128:192.168.203.143@tcp:250:kworker.0 00000100:00000001:3.0:1787829302.119038:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1787829302.119044:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:1.0:1787829302.119045:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 02000000:00000001:3.0:1787829302.119047:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829302.119051:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829302.119052:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:1.0:1787829302.119058:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829302.119058:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1787829302.119063:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829302.119070:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:1.0:1787829302.119071:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1787829302.119074:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787829302.119077:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:3.0:1787829302.119080:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00080000:1.0:1787829302.119082:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4102 00000100:00000001:3.0:1787829302.119083:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839541340992 : -126234168210624 : ffff8d30d13ecb40) 00000100:00000001:3.0:1787829302.119088:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00080000:1.0:1787829302.119091:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1787829302.119094:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787829302.119095:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:1.0:1787829302.119097:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:1.0:1787829302.119098:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:1.0:1787829302.119100:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000020:00000001:1.0:1787829302.119102:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:1.0:1787829302.119104:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787829302.119105:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:1.0:1787829302.119108:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:1.0:1787829302.119110:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829302.119111:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1787829302.119114:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829302.119115:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829302.119116:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:1.0:1787829302.119118:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1787829302.119120:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829302.119121:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829302.119126:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1787829302.119127:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829302.119129:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1787829302.119130:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1787829302.119131:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829302.119136:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:3.0:1787829302.119137:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00100000:1.0:1787829302.119144:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d13ed680 x1874670646816384/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787829302.119150:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787829302.119152:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829302.119154:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1787829302.119157:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1787829302.119158:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00080000:1.0:1787829302.119159:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1787829302.119163:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:3.0:1787829302.119164:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00080000:1.0:1787829302.119165:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787829302.119169:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829302.119170:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:3.0:1787829302.119173:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00080000:1.0:1787829302.119175:0:468:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4102 00000100:00000001:3.0:1787829302.119180:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1787829302.119181:0:468:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1787829302.119184:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829302.119185:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4298776320 00000100:00000001:1.0:1787829302.119186:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:1.0:1787829302.119190:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:1.0:1787829302.119193:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:1.0:1787829302.119194:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000020:00000001:1.0:1787829302.119196:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:1.0:1787829302.119197:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787829302.119198:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:1.0:1787829302.119200:0:468:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:1.0:1787829302.119201:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829302.119202:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:3.0:1787829302.119204:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 02000000:00000001:1.0:1787829302.119205:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829302.119210:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829302.119212:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:1.0:1787829302.119213:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000008:00000001:1.0:1787829302.119216:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829302.119217:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829302.119218:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4298776320 02000000:00000001:1.0:1787829302.119221:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1787829302.119224:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829302.119225:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1787829302.119226:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:3.0:1787829302.119227:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 02000000:00000001:1.0:1787829302.119228:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829302.119235:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829302.119236:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829302.119241:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829302.119244:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d13ed2c0 x1874670646816512/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787829302.119245:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00000001:1.0:1787829302.119252:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00100000:3.0:1787829302.119253:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829302/real 1787829302] req@ffff8d30d13ecb40 x1874670646816128/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829327 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787829302.119255:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829302.119259:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:3.0:1787829302.119270:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787829302.119272:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829302.119276:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829302.119278:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829302.119280:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829302.119282:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829302.119284:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829302.119288:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d13ecb40 x1874670646816128/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829327 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829302.119313:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829302.119316:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829302.119318:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829302.119320:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829302.119321:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829302.119323:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829302.119324:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829302.119327:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829302.119330:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829302.119331:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829302.119334:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d13ecb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646816128:192.168.203.143@tcp:250:kworker.0 00000100:00000001:3.0:1787829302.119339:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829302.119340:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829302.119342:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829302.119345:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829302.119348:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829302.119350:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829302.119352:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829302.119357:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829302.119358:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829302.119361:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829302.119362:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829302.119364:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829302.119372:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787829302.119374:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829302.119375:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1787829302.119376:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787829302.119377:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787829302.119379:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787829302.119380:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829302.119381:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787829302.119383:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829302.119385:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829302.119387:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d13ef0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646816256:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829302.119389:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1787829302.119391:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1787829302.119392:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829302.119393:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1787829302.119395:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829302.119399:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787829302.119400:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839541350592 : -126234168201024 : ffff8d30d13ef0c0) 00000100:00000001:3.0:1787829302.119405:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1787829302.119414:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:3.0:1787829302.119418:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787829302.119420:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829302.119421:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829302.119422:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1787829302.119424:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829302.119427:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298775062 00000100:00000001:3.0:1787829302.119430:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298776320 00000100:00000001:3.0:1787829302.119431:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298775062 00000100:00000001:3.0:1787829302.119433:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298776320 00000100:00000001:3.0:1787829302.119435:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829302.119437:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829302.119438:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1787829302.119441:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829302/real 1787829302] req@ffff8d30d13ef0c0 x1874670646816256/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829327 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1787829302.119446:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787829302.119447:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829302.119448:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829302.119449:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829302.119450:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829302.119451:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829302.119452:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829302.119455:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d13ef0c0 x1874670646816256/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829327 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829302.119460:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829302.119461:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829302.119462:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829302.119463:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829302.119464:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829302.119465:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829302.119466:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829302.119468:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829302.119470:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829302.119471:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829302.119473:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d13ef0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646816256:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829302.119475:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787829302.119476:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787829302.119477:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829302.119478:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787829302.119491:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829302.119492:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829302.119494:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d13ed680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646816384:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829302.119497:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1787829302.119498:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1787829302.119499:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829302.119500:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1787829302.119501:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829302.119504:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787829302.119505:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839541343872 : -126234168207744 : ffff8d30d13ed680) 00000100:00000001:3.0:1787829302.119507:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1787829302.119512:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:3.0:1787829302.119514:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787829302.119516:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829302.119517:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829302.119518:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1787829302.119520:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829302.119522:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298775062 00000100:00000001:3.0:1787829302.119524:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298776320 00000100:00000001:3.0:1787829302.119525:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298775062 00000100:00000001:3.0:1787829302.119527:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298776320 00000100:00000001:3.0:1787829302.119528:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829302.119530:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829302.119531:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1787829302.119533:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829302/real 1787829302] req@ffff8d30d13ed680 x1874670646816384/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829327 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1787829302.119537:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787829302.119538:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829302.119539:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829302.119540:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829302.119541:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829302.119542:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829302.119543:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829302.119545:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d13ed680 x1874670646816384/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829327 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829302.119550:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829302.119552:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829302.119553:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829302.119554:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829302.119555:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829302.119556:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829302.119557:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829302.119558:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829302.119560:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829302.119561:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829302.119563:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d13ed680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646816384:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829302.119565:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787829302.119566:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787829302.119566:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829302.119568:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787829302.119569:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829302.119570:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829302.119571:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d13ed2c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646816512:192.168.203.143@tcp:8:kworker.0 00000100:00000001:3.0:1787829302.119573:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1787829302.119574:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1787829302.119575:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829302.119576:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1787829302.119578:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829302.119581:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787829302.119582:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839541342912 : -126234168208704 : ffff8d30d13ed2c0) 00000100:00000001:3.0:1787829302.119584:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1787829302.119589:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:3.0:1787829302.119591:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787829302.119592:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829302.119594:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829302.119595:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1787829302.119596:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829302.119598:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298775075 00000100:00000001:3.0:1787829302.119637:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298776320 00000100:00000001:3.0:1787829302.119639:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298775076 00000100:00000001:3.0:1787829302.119641:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298776320 00000100:00000001:3.0:1787829302.119642:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829302.119644:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829302.119645:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1787829302.119647:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829302/real 1787829302] req@ffff8d30d13ed2c0 x1874670646816512/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829327 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1787829302.119653:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787829302.119654:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829302.119656:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829302.119657:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829302.119668:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829302.119669:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829302.119670:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829302.119673:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d13ed2c0 x1874670646816512/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829327 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829302.119677:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829302.119678:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829302.119680:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829302.119681:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829302.119681:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829302.119683:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829302.119684:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829302.119685:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829302.119687:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829302.119689:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829302.119692:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d13ed2c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646816512:192.168.203.143@tcp:8:kworker.0 00000100:00000001:3.0:1787829302.119694:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829302.119696:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829302.119697:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829302.119698:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829302.119699:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829302.119700:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829302.119701:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829302.119703:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829302.119704:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829302.119706:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829302.119707:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829302.119708:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829302.119709:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829302.119710:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829302.119711:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829302.119712:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829302.119713:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829302.119714:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829302.119714:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829302.119716:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829302.119717:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829302.119718:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829302.119718:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829302.119719:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829302.119720:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829302.119721:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829302.119721:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829302.119722:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829302.119723:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829302.119724:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829302.119724:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829302.119726:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829302.119728:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787829302.119728:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829302.119730:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1787829302.119731:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829303.014754:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829303.014761:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829303.014822:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829303.014826:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829303.014829:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829304.038721:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829304.038736:0:815:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1787829304.038740:0:815:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1787829304.038761:0:815:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829304.038768:0:815:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1787829304.038771:0:815:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829304.038775:0:815:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829304.038779:0:815:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829304.038843:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829304.038848:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829304.038851:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829305.062760:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829305.062769:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829305.062908:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829305.063024:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829305.063028:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829306.086745:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829306.086755:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829306.086821:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829306.086825:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829306.086828:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829307.110759:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829307.110767:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829307.110831:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829307.110836:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829307.110839:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787829307.238762:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829307.238772:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1787829307.238775:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787829307.238779:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787829307.238784:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787829307.238792:0:40701:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 4107 00000100:00080000:0.0:1787829307.238805:0:40701:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787829307.238808:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829307.238809:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787829307.238814:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787829307.238825:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787829307.238828:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:0.0:1787829307.238830:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:0.0:1787829307.238832:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829307.238833:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:0.0:1787829307.238837:0:40701:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787829307.238839:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829307.238841:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787829307.238845:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787829307.238848:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829307.238850:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1787829307.238852:0:40701:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829307.238866:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787829307.238868:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829307.238872:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787829307.238874:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787829307.238879:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829307.238893:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787829307.238901:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d24000 x1874670646816640/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787829307.238911:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787829307.238915:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829307.238927:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787829307.238930:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829307.238932:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1787829307.238934:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787829307.238935:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787829307.238937:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787829307.238939:0:40701:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4107 00000100:00080000:0.0:1787829307.238941:0:40701:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787829307.238942:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829307.238944:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787829307.238946:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787829307.238947:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787829307.238949:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000020:00000001:0.0:1787829307.238951:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:0.0:1787829307.238952:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829307.238953:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:0.0:1787829307.238955:0:40701:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:3.0:1787829307.238955:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787829307.238960:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829307.238971:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 02000000:00000001:0.0:1787829307.238972:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:3.0:1787829307.238977:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 02000000:00000001:0.0:1787829307.238980:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829307.238981:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 02000000:00000001:0.0:1787829307.238985:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829307.238985:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829307.238990:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 02000000:00000001:3.0:1787829307.238993:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000008:00100000:0.0:1787829307.238996:0:40701:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 02000000:00000001:3.0:1787829307.238997:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000008:00000001:0.0:1787829307.239002:0:40701:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829307.239003:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829307.239005:0:40701:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829307.239010:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00100000:3.0:1787829307.239011:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d24000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646816640:192.168.203.143@tcp:250:kworker.0 02000000:00000001:0.0:1787829307.239012:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829307.239017:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:3.0:1787829307.239019:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787829307.239020:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787829307.239024:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829307.239026:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:0.0:1787829307.239028:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829307.239030:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829307.239033:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00100000:0.0:1787829307.239036:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d24f00 x1874670646816768/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:3.0:1787829307.239039:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829307.239046:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787829307.239048:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829307.239051:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787829307.239053:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829307.239056:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1787829307.239057:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787829307.239058:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787829307.239060:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:3.0:1787829307.239061:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00080000:0.0:1787829307.239063:0:40701:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4107 00000100:00000001:3.0:1787829307.239064:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601336320 : -126234108215296 : ffff8d30d4d24000) 00000100:00080000:0.0:1787829307.239068:0:40701:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787829307.239070:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:0.0:1787829307.239071:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829307.239074:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787829307.239076:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787829307.239077:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787829307.239079:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000020:00000001:0.0:1787829307.239080:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:0.0:1787829307.239082:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829307.239083:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:0.0:1787829307.239085:0:40701:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787829307.239087:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829307.239089:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787829307.239090:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787829307.239091:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829307.239093:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1787829307.239094:0:40701:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1787829307.239096:0:40701:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829307.239097:0:40701:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829307.239101:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787829307.239102:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829307.239103:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787829307.239104:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787829307.239105:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829307.239109:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787829307.239112:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d25a40 x1874670646816896/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1787829307.239116:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:0.0:1787829307.239124:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787829307.239126:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829307.239128:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787829307.239130:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829307.239133:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1787829307.239134:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787829307.239136:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787829307.239136:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1787829307.239138:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:3.0:1787829307.239142:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829307.239145:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829307.239147:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00080000:0.0:1787829307.239148:0:40701:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4107 00000100:00000001:3.0:1787829307.239152:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787829307.239153:0:40701:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787829307.239157:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829307.239158:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:3.0:1787829307.239159:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4298781440 00000020:00000001:0.0:1787829307.239160:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787829307.239162:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787829307.239163:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000020:00000001:0.0:1787829307.239165:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:0.0:1787829307.239167:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829307.239168:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:0.0:1787829307.239173:0:40701:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787829307.239174:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829307.239175:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787829307.239177:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787829307.239178:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829307.239180:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:3.0:1787829307.239180:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 00000008:00100000:0.0:1787829307.239182:0:40701:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1787829307.239185:0:40701:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829307.239186:0:40701:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829307.239189:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4298781440 02000000:00000001:0.0:1787829307.239190:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787829307.239193:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829307.239195:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787829307.239196:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787829307.239197:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829307.239197:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 00000100:00000001:3.0:1787829307.239203:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787829307.239205:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829307.239207:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829307.239211:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787829307.239212:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d270c0 x1874670646817024/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787829307.239218:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829307/real 1787829307] req@ffff8d30d4d24000 x1874670646816640/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829332 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787829307.239220:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787829307.239224:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829307.239229:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:3.0:1787829307.239237:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787829307.239239:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829307.239242:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829307.239245:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829307.239246:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829307.239249:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829307.239251:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829307.239255:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d24000 x1874670646816640/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829332 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829307.239261:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829307.239264:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829307.239266:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829307.239267:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829307.239268:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829307.239270:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829307.239272:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829307.239274:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829307.239277:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829307.239278:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829307.239281:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d24000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646816640:192.168.203.143@tcp:250:kworker.0 00000100:00000001:3.0:1787829307.239285:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829307.239287:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829307.239288:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829307.239290:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829307.239293:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829307.239295:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829307.239297:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829307.239301:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829307.239304:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829307.239307:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829307.239309:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829307.239312:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829307.239321:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787829307.239323:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829307.239326:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1787829307.239327:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787829307.239329:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787829307.239331:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787829307.239332:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829307.239334:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787829307.239337:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829307.239339:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829307.239343:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d24f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646816768:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829307.239347:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1787829307.239349:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1787829307.239351:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829307.239353:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1787829307.239356:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829307.239361:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787829307.239362:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601340160 : -126234108211456 : ffff8d30d4d24f00) 00000100:00000001:3.0:1787829307.239367:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1787829307.239377:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:3.0:1787829307.239381:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787829307.239383:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829307.239384:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829307.239385:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1787829307.239387:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829307.239391:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298780182 00000100:00000001:3.0:1787829307.239393:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298781440 00000100:00000001:3.0:1787829307.239395:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298780182 00000100:00000001:3.0:1787829307.239397:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298781440 00000100:00000001:3.0:1787829307.239399:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829307.239401:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829307.239402:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1787829307.239405:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829307/real 1787829307] req@ffff8d30d4d24f00 x1874670646816768/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829332 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1787829307.239413:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787829307.239415:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829307.239417:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829307.239419:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829307.239420:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829307.239422:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829307.239423:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829307.239428:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d24f00 x1874670646816768/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829332 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829307.239436:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829307.239439:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829307.239442:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829307.239444:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829307.239445:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829307.239447:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829307.239449:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829307.239452:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829307.239456:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829307.239458:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829307.239461:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d24f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646816768:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829307.239464:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787829307.239465:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787829307.239467:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829307.239468:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787829307.239470:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829307.239471:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829307.239473:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d25a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646816896:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829307.239476:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1787829307.239477:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1787829307.239478:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829307.239479:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1787829307.239482:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829307.239485:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787829307.239487:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601343040 : -126234108208576 : ffff8d30d4d25a40) 00000100:00000001:3.0:1787829307.239490:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1787829307.239516:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:3.0:1787829307.239520:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787829307.239521:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829307.239522:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829307.239524:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1787829307.239526:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829307.239528:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298780182 00000100:00000001:3.0:1787829307.239531:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298781440 00000100:00000001:3.0:1787829307.239533:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298780182 00000100:00000001:3.0:1787829307.239534:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298781440 00000100:00000001:3.0:1787829307.239536:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829307.239537:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829307.239538:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1787829307.239541:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829307/real 1787829307] req@ffff8d30d4d25a40 x1874670646816896/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829332 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1787829307.239547:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787829307.239548:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829307.239549:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829307.239550:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829307.239551:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829307.239553:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829307.239554:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829307.239557:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d25a40 x1874670646816896/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829332 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829307.239562:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829307.239564:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829307.239566:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829307.239567:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829307.239567:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829307.239569:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829307.239570:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829307.239571:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829307.239573:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829307.239575:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829307.239577:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d25a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646816896:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829307.239579:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787829307.239580:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787829307.239581:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829307.239582:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787829307.239583:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829307.239600:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829307.239603:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d270c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646817024:192.168.203.143@tcp:8:kworker.0 00000100:00000001:3.0:1787829307.239606:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1787829307.239608:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1787829307.239609:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829307.239610:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1787829307.239613:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829307.239617:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787829307.239618:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601348800 : -126234108202816 : ffff8d30d4d270c0) 00000100:00000001:3.0:1787829307.239621:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1787829307.239658:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:3.0:1787829307.239663:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787829307.239665:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829307.239667:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829307.239668:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1787829307.239671:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829307.239674:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298780196 00000100:00000001:3.0:1787829307.239676:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298781440 00000100:00000001:3.0:1787829307.239678:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298780196 00000100:00000001:3.0:1787829307.239680:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298781440 00000100:00000001:3.0:1787829307.239682:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829307.239683:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829307.239685:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1787829307.239688:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829307/real 1787829307] req@ffff8d30d4d270c0 x1874670646817024/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829332 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1787829307.239694:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787829307.239695:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829307.239706:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829307.239708:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829307.239709:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829307.239711:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829307.239712:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829307.239716:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d270c0 x1874670646817024/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829332 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829307.239721:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829307.239723:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829307.239725:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829307.239726:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829307.239728:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829307.239729:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829307.239731:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829307.239733:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829307.239735:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829307.239736:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829307.239739:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d270c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646817024:192.168.203.143@tcp:8:kworker.0 00000100:00000001:3.0:1787829307.239743:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829307.239744:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829307.239746:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829307.239747:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829307.239749:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829307.239750:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829307.239751:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829307.239754:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829307.239756:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829307.239759:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829307.239761:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829307.239763:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829307.239765:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829307.239766:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829307.239769:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829307.239770:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829307.239771:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829307.239773:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829307.239775:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829307.239777:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829307.239779:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829307.239781:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829307.239782:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829307.239784:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829307.239801:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829307.239802:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829307.239804:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829307.239805:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829307.239806:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829307.239810:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829307.239811:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829307.239813:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829307.239817:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787829307.239819:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829307.239821:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1787829307.239823:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829308.134681:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829308.134687:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829308.134729:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829308.134733:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829308.134735:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829309.158667:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829309.158675:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829309.158730:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829309.158733:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829309.158736:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829310.182726:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829310.182734:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829310.182818:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829310.182825:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829310.182831:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829311.206699:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829311.206707:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829311.206774:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829311.206778:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829311.206782:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829312.230697:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829312.230703:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829312.230752:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829312.230755:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829312.230758:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787829312.358707:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829312.358715:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1787829312.358717:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1787829312.358720:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787829312.358725:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1787829312.358732:0:40701:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 4112 00000100:00080000:1.0:1787829312.358741:0:40701:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1787829312.358744:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787829312.358746:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:1.0:1787829312.358749:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:1.0:1787829312.358751:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:1.0:1787829312.358754:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:1.0:1787829312.358756:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:1.0:1787829312.358758:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787829312.358759:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:1.0:1787829312.358763:0:40701:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:1.0:1787829312.358765:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829312.358767:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1787829312.358770:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829312.358772:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829312.358774:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:1.0:1787829312.358776:0:40701:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829312.358786:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1787829312.358788:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829312.358790:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1787829312.358792:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1787829312.358796:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829312.358805:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829312.358812:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d13efc00 x1874670646817152/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787829312.358820:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787829312.358823:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829312.358835:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1787829312.358837:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829312.358839:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1787829312.358841:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1787829312.358842:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787829312.358844:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1787829312.358845:0:40701:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4112 00000100:00080000:1.0:1787829312.358847:0:40701:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1787829312.358849:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787829312.358850:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:1.0:1787829312.358851:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:1.0:1787829312.358852:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:1.0:1787829312.358854:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000020:00000001:1.0:1787829312.358856:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:1.0:1787829312.358858:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787829312.358859:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:1.0:1787829312.358861:0:40701:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:1.0:1787829312.358863:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829312.358865:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:3.0:1787829312.358865:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:1.0:1787829312.358871:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829312.358872:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 02000000:00000001:1.0:1787829312.358876:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829312.358878:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:3.0:1787829312.358878:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000008:00100000:1.0:1787829312.358884:0:40701:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00000001:3.0:1787829312.358884:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000008:00000001:1.0:1787829312.358888:0:40701:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829312.358888:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829312.358894:0:40701:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829312.358897:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787829312.358897:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787829312.358901:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829312.358904:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787829312.358904:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829312.358907:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1787829312.358909:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829312.358911:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829312.358914:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829312.358920:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d13efc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646817152:192.168.203.143@tcp:250:kworker.0 00000100:00100000:1.0:1787829312.358922:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d13ede00 x1874670646817280/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787829312.358945:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:1.0:1787829312.358948:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787829312.358951:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829312.358953:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:1.0:1787829312.358954:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1787829312.358959:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 02000000:00000001:3.0:1787829312.358959:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1787829312.358965:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:1.0:1787829312.358967:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 02000000:00000001:3.0:1787829312.358967:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00080000:1.0:1787829312.358970:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787829312.358973:0:40701:0:(import.c:485:import_select_connection()) Process entered 02000000:00000001:3.0:1787829312.358975:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1787829312.358976:0:40701:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4112 00000100:00080000:1.0:1787829312.358979:0:40701:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1787829312.358981:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787829312.358981:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:1.0:1787829312.358983:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:1.0:1787829312.358984:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:1.0:1787829312.358987:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000020:00000001:1.0:1787829312.358990:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:1.0:1787829312.358991:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787829312.358993:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:1.0:1787829312.358996:0:40701:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:1.0:1787829312.358998:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829312.358999:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1787829312.359001:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829312.359003:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829312.359003:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787829312.359007:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:3.0:1787829312.359008:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839541353472 : -126234168198144 : ffff8d30d13efc00) 00000008:00100000:1.0:1787829312.359009:0:40701:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1787829312.359015:0:40701:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829312.359016:0:40701:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829312.359017:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 02000000:00000001:1.0:1787829312.359020:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1787829312.359023:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829312.359024:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1787829312.359025:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1787829312.359027:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829312.359031:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829312.359035:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d13eed00 x1874670646817408/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787829312.359041:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787829312.359043:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829312.359045:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1787829312.359047:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829312.359049:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1787829312.359050:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1787829312.359051:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787829312.359052:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1787829312.359054:0:40701:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4112 00000100:00080000:1.0:1787829312.359056:0:40701:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1787829312.359057:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787829312.359058:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:1.0:1787829312.359059:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:1.0:1787829312.359061:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:1.0:1787829312.359062:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000020:00000001:1.0:1787829312.359066:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000800:00000100:3.0:1787829312.359066:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787829312.359070:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787829312.359072:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:1.0:1787829312.359074:0:40701:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:1.0:1787829312.359076:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829312.359078:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1787829312.359080:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829312.359081:0:2390:0:(events.c:34:request_out_callback()) Process entered 02000000:00000001:1.0:1787829312.359082:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829312.359085:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:1.0:1787829312.359086:0:40701:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00000001:3.0:1787829312.359087:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000008:00000001:1.0:1787829312.359089:0:40701:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829312.359090:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829312.359092:0:40701:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829312.359094:0:2390:0:(events.c:68:request_out_callback()) Process leaving 02000000:00000001:1.0:1787829312.359097:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1787829312.359099:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829312.359101:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:3.0:1787829312.359101:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829312.359104:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1787829312.359106:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829312.359107:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4298786560 00000100:00000001:1.0:1787829312.359110:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829312.359115:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d13ef840 x1874670646817536/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787829312.359118:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 00000100:00000001:1.0:1787829312.359120:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787829312.359123:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829312.359125:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829312.359125:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4298786560 00000100:00000001:3.0:1787829312.359131:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 00000100:00000001:3.0:1787829312.359136:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829312.359138:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829312.359140:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1787829312.359151:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829312/real 1787829312] req@ffff8d30d13efc00 x1874670646817152/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829337 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1787829312.359163:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787829312.359165:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829312.359169:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829312.359172:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829312.359174:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829312.359176:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829312.359179:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829312.359184:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d13efc00 x1874670646817152/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829337 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829312.359193:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829312.359196:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829312.359200:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829312.359202:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829312.359203:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829312.359206:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829312.359208:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829312.359211:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829312.359215:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829312.359218:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829312.359222:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d13efc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646817152:192.168.203.143@tcp:250:kworker.0 00000100:00000001:3.0:1787829312.359227:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829312.359229:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829312.359231:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829312.359234:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829312.359237:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829312.359239:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829312.359241:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829312.359245:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829312.359246:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829312.359249:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829312.359251:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829312.359253:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829312.359260:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787829312.359262:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829312.359263:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1787829312.359264:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787829312.359266:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787829312.359267:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787829312.359268:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829312.359270:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787829312.359271:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829312.359273:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829312.359276:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d13ede00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646817280:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829312.359280:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1787829312.359282:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1787829312.359284:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829312.359285:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1787829312.359288:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829312.359295:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787829312.359297:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839541345792 : -126234168205824 : ffff8d30d13ede00) 00000100:00000001:3.0:1787829312.359302:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1787829312.359316:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:3.0:1787829312.359323:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787829312.359325:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829312.359327:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829312.359329:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1787829312.359331:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829312.359336:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298785302 00000100:00000001:3.0:1787829312.359340:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298786560 00000100:00000001:3.0:1787829312.359342:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298785302 00000100:00000001:3.0:1787829312.359345:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298786560 00000100:00000001:3.0:1787829312.359348:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829312.359350:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829312.359352:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1787829312.359357:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829312/real 1787829312] req@ffff8d30d13ede00 x1874670646817280/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829337 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1787829312.359366:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787829312.359367:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829312.359370:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829312.359372:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829312.359374:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829312.359376:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829312.359377:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829312.359381:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d13ede00 x1874670646817280/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829337 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829312.359389:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829312.359391:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829312.359394:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829312.359396:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829312.359397:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829312.359399:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829312.359400:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829312.359403:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829312.359406:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829312.359408:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829312.359412:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d13ede00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646817280:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829312.359416:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787829312.359417:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787829312.359419:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829312.359421:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787829312.359423:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829312.359425:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829312.359428:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d13eed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646817408:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829312.359432:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1787829312.359434:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1787829312.359435:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829312.359437:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1787829312.359440:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829312.359445:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787829312.359447:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839541349632 : -126234168201984 : ffff8d30d13eed00) 00000100:00000001:3.0:1787829312.359450:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1787829312.359463:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:3.0:1787829312.359468:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787829312.359470:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829312.359471:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829312.359473:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1787829312.359476:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829312.359480:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298785302 00000100:00000001:3.0:1787829312.359482:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298786560 00000100:00000001:3.0:1787829312.359484:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298785302 00000100:00000001:3.0:1787829312.359486:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298786560 00000100:00000001:3.0:1787829312.359488:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829312.359490:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829312.359492:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1787829312.359496:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829312/real 1787829312] req@ffff8d30d13eed00 x1874670646817408/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829337 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1787829312.359504:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787829312.359506:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829312.359509:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829312.359510:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829312.359512:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829312.359514:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829312.359515:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829312.359520:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d13eed00 x1874670646817408/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829337 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829312.359527:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829312.359529:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829312.359532:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829312.359534:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829312.359535:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829312.359537:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829312.359538:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829312.359541:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829312.359544:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829312.359546:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829312.359550:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d13eed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646817408:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829312.359554:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787829312.359555:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787829312.359557:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829312.359559:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787829312.359560:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829312.359562:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829312.359565:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d13ef840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646817536:192.168.203.143@tcp:8:kworker.0 00000100:00000001:3.0:1787829312.359569:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1787829312.359570:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1787829312.359572:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829312.359574:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1787829312.359577:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829312.359582:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787829312.359583:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839541352512 : -126234168199104 : ffff8d30d13ef840) 00000100:00000001:3.0:1787829312.359586:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1787829312.359597:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:3.0:1787829312.359636:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787829312.359638:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829312.359640:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829312.359641:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1787829312.359643:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829312.359653:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298785316 00000100:00000001:3.0:1787829312.359655:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298786560 00000100:00000001:3.0:1787829312.359657:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298785316 00000100:00000001:3.0:1787829312.359658:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298786560 00000100:00000001:3.0:1787829312.359659:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829312.359661:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829312.359662:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1787829312.359665:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829312/real 1787829312] req@ffff8d30d13ef840 x1874670646817536/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829337 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1787829312.359671:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787829312.359672:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829312.359673:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829312.359674:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829312.359675:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829312.359677:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829312.359678:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829312.359680:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d13ef840 x1874670646817536/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829337 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829312.359685:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829312.359686:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829312.359687:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829312.359688:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829312.359689:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829312.359690:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829312.359691:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829312.359693:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829312.359694:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829312.359696:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829312.359698:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d13ef840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646817536:192.168.203.143@tcp:8:kworker.0 00000100:00000001:3.0:1787829312.359701:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829312.359702:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829312.359703:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829312.359704:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829312.359706:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829312.359707:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829312.359708:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829312.359709:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829312.359710:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829312.359712:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829312.359713:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829312.359715:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829312.359715:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829312.359716:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829312.359717:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829312.359718:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829312.359719:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829312.359721:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829312.359721:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829312.359722:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829312.359723:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829312.359724:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829312.359725:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829312.359725:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829312.359727:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829312.359727:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829312.359728:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829312.359729:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829312.359729:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829312.359730:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829312.359731:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829312.359732:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829312.359734:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787829312.359735:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829312.359736:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1787829312.359738:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829313.254693:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829313.254698:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829313.254753:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829313.254758:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829313.254761:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829314.278667:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829314.278673:0:815:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1787829314.278676:0:815:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1787829314.278678:0:815:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829314.278682:0:815:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1787829314.278683:0:815:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829314.278685:0:815:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829314.278687:0:815:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829314.278725:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829314.278728:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829314.278730:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829315.302704:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829315.302710:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829315.302807:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829315.302814:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829315.302819:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829316.326716:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829316.326722:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829316.326770:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829316.326774:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829316.326777:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829317.350727:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829317.350735:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829317.350786:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829317.350790:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829317.350793:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787829317.478681:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829317.478692:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1787829317.478694:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1787829317.478698:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787829317.478702:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1787829317.478710:0:468:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 4117 00000100:00080000:1.0:1787829317.478719:0:468:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1787829317.478724:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787829317.478726:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:1.0:1787829317.478730:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:1.0:1787829317.478733:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:1.0:1787829317.478736:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:1.0:1787829317.478739:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:1.0:1787829317.478741:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787829317.478743:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:1.0:1787829317.478748:0:468:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:1.0:1787829317.478751:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829317.478754:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1787829317.478757:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829317.478761:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829317.478764:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:1.0:1787829317.478766:0:468:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829317.478777:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1787829317.478778:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829317.478782:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1787829317.478785:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1787829317.478789:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829317.478799:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829317.478808:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d13ef480 x1874670646817664/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787829317.478820:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787829317.478823:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829317.478837:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1787829317.478840:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829317.478844:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1787829317.478845:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1787829317.478847:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787829317.478850:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1787829317.478853:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4117 00000100:00080000:1.0:1787829317.478857:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1787829317.478859:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787829317.478860:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:1.0:1787829317.478862:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:1.0:1787829317.478863:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:1.0:1787829317.478865:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000020:00000001:1.0:1787829317.478867:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:3.0:1787829317.478867:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787829317.478871:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787829317.478873:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:3.0:1787829317.478874:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00080000:1.0:1787829317.478877:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:3.0:1787829317.478878:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787829317.478880:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829317.478881:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 02000000:00000001:1.0:1787829317.478882:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:3.0:1787829317.478884:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829317.478888:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829317.478901:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787829317.478902:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829317.478905:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 02000000:00000001:3.0:1787829317.478906:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000008:00100000:1.0:1787829317.478909:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 02000000:00000001:3.0:1787829317.478910:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:1.0:1787829317.478914:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829317.478917:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829317.478919:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d13ef480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646817664:192.168.203.143@tcp:250:kworker.0 02000000:00000001:1.0:1787829317.478921:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1787829317.478924:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829317.478925:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:1.0:1787829317.478926:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1787829317.478929:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787829317.478931:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1787829317.478934:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829317.478937:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787829317.478940:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829317.478941:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829317.478951:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829317.478958:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00100000:1.0:1787829317.478960:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d13ee580 x1874670646817792/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787829317.478961:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839541351552 : -126234168200064 : ffff8d30d13ef480) 00000100:00000001:3.0:1787829317.478966:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:1.0:1787829317.478975:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787829317.478978:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829317.478982:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1787829317.478985:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829317.478989:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:1.0:1787829317.478991:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1787829317.478994:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787829317.478996:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1787829317.478999:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4117 00000100:00080000:1.0:1787829317.479004:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000800:00000100:3.0:1787829317.479007:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787829317.479008:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787829317.479012:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:1.0:1787829317.479015:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:1.0:1787829317.479016:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:1.0:1787829317.479019:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000020:00000001:1.0:1787829317.479021:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:1.0:1787829317.479024:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829317.479024:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787829317.479028:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:3.0:1787829317.479031:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00080000:1.0:1787829317.479035:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:3.0:1787829317.479035:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829317.479040:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829317.479041:0:2390:0:(events.c:68:request_out_callback()) Process leaving 02000000:00000001:1.0:1787829317.479045:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:3.0:1787829317.479048:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829317.479049:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829317.479054:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829317.479055:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4298791680 00000100:00000001:1.0:1787829317.479057:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:1.0:1787829317.479061:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1787829317.479064:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829317.479065:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 00000100:00000001:1.0:1787829317.479066:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829317.479073:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4298791680 02000000:00000001:1.0:1787829317.479074:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1787829317.479077:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829317.479080:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:3.0:1787829317.479081:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 02000000:00000001:1.0:1787829317.479084:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:3.0:1787829317.479085:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829317.479088:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829317.479088:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829317.479094:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00000001:1.0:1787829317.479098:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829317.479100:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829317/real 1787829317] req@ffff8d30d13ef480 x1874670646817664/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829342 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829317.479108:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d13ee1c0 x1874670646817920/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000001:3.0:1787829317.479112:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787829317.479114:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829317.479117:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829317.479120:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829317.479123:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829317.479124:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787829317.479125:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829317.479129:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829317.479130:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829317.479137:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829317.479138:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d13ef480 x1874670646817664/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829342 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829317.479143:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829317.479151:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1787829317.479153:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829317.479154:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:3.0:1787829317.479157:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787829317.479158:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787829317.479160:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829317.479164:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:3.0:1787829317.479164:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829317.479168:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829317.479170:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00080000:1.0:1787829317.479172:0:468:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4117 00000100:00000001:3.0:1787829317.479173:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829317.479177:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829317.479181:0:468:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829317.479182:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787829317.479186:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829317.479187:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000001:1.0:1787829317.479191:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00100000:3.0:1787829317.479193:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d13ef480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646817664:192.168.203.143@tcp:250:kworker.0 00000020:00000001:1.0:1787829317.479196:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:3.0:1787829317.479198:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:1.0:1787829317.479199:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:3.0:1787829317.479203:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000020:00000001:1.0:1787829317.479204:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:3.0:1787829317.479206:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 00000020:00000001:1.0:1787829317.479209:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 02000000:00000001:3.0:1787829317.479210:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 00000100:00000001:1.0:1787829317.479214:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 02000000:00000001:3.0:1787829317.479215:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00000001:1.0:1787829317.479218:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:3.0:1787829317.479218:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829317.479221:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829317.479225:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 00000100:00080000:1.0:1787829317.479226:0:468:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 02000000:00000001:3.0:1787829317.479227:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787829317.479231:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829317.479232:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 02000000:00000001:1.0:1787829317.479236:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:3.0:1787829317.479236:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829317.479240:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:1.0:1787829317.479241:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829317.479246:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829317.479246:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787829317.479248:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:3.0:1787829317.479249:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000008:00100000:1.0:1787829317.479253:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00000001:3.0:1787829317.479253:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1787829317.479255:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787829317.479258:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000008:00000001:1.0:1787829317.479259:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829317.479260:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787829317.479263:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829317.479264:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829317.479269:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787829317.479271:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829317.479272:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787829317.479273:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829317.479276:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829317.479278:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d13ee580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646817792:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787829317.479280:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1787829317.479284:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:3.0:1787829317.479285:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787829317.479290:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829317.479290:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1787829317.479293:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829317.479295:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:1.0:1787829317.479298:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829317.479298:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829317.479307:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787829317.479308:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839541347712 : -126234168203904 : ffff8d30d13ee580) 00000100:00100000:1.0:1787829317.479309:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d13eda40 x1874670646818048/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787829317.479313:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:1.0:1787829317.479320:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787829317.479323:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:3.0:1787829317.479327:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787829317.479329:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829317.479332:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787829317.479336:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829317.479337:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829317.479338:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1787829317.479341:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829317.479343:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298790422 00000100:00000001:3.0:1787829317.479346:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298791680 00000100:00000001:3.0:1787829317.479348:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298790422 00000100:00000001:3.0:1787829317.479350:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298791680 00000100:00000001:3.0:1787829317.479352:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829317.479353:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829317.479355:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1787829317.479358:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829317/real 1787829317] req@ffff8d30d13ee580 x1874670646817792/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829342 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1787829317.479363:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787829317.479364:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829317.479366:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829317.479367:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829317.479368:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829317.479369:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829317.479370:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829317.479372:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d13ee580 x1874670646817792/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829342 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829317.479377:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829317.479378:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829317.479380:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829317.479381:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829317.479381:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829317.479382:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829317.479384:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829317.479385:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829317.479387:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829317.479388:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829317.479390:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d13ee580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646817792:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829317.479392:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787829317.479393:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787829317.479394:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829317.479395:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787829317.479397:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829317.479398:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829317.479400:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d13ee1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646817920:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829317.479402:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1787829317.479403:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1787829317.479404:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829317.479405:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1787829317.479407:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829317.479410:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787829317.479410:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839541346752 : -126234168204864 : ffff8d30d13ee1c0) 00000100:00000001:3.0:1787829317.479412:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1787829317.479418:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:3.0:1787829317.479420:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787829317.479422:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829317.479423:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829317.479424:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1787829317.479425:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829317.479427:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298790422 00000100:00000001:3.0:1787829317.479429:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298791680 00000100:00000001:3.0:1787829317.479430:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298790422 00000100:00000001:3.0:1787829317.479432:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298791680 00000100:00000001:3.0:1787829317.479433:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829317.479434:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829317.479435:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1787829317.479438:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829317/real 1787829317] req@ffff8d30d13ee1c0 x1874670646817920/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829342 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1787829317.479442:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787829317.479443:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829317.479444:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829317.479445:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829317.479446:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829317.479447:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829317.479447:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829317.479450:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d13ee1c0 x1874670646817920/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829342 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829317.479454:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829317.479456:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829317.479457:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829317.479458:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829317.479459:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829317.479460:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829317.479461:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829317.479462:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829317.479464:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829317.479465:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829317.479467:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d13ee1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646817920:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829317.479469:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829317.479470:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829317.479471:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829317.479472:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829317.479473:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829317.479473:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829317.479474:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829317.479476:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829317.479477:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829317.479478:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829317.479479:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829317.479480:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829317.479481:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829317.479481:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829317.479482:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829317.479483:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829317.479484:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829317.479485:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829317.479486:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829317.479487:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829317.479487:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829317.479489:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829317.479491:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787829317.479492:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829317.479493:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1787829317.479494:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787829317.479495:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787829317.479496:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787829317.479497:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829317.479498:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787829317.479499:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829317.479501:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829317.479502:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d13eda40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646818048:192.168.203.143@tcp:8:kworker.0 00000100:00000001:3.0:1787829317.479504:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1787829317.479505:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1787829317.479506:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829317.479508:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1787829317.479509:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829317.479512:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787829317.479513:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839541344832 : -126234168206784 : ffff8d30d13eda40) 00000100:00000001:3.0:1787829317.479515:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1787829317.479521:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:3.0:1787829317.479524:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787829317.479525:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829317.479526:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829317.479527:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1787829317.479529:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829317.479531:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298790435 00000100:00000001:3.0:1787829317.479532:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298791680 00000100:00000001:3.0:1787829317.479534:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298790435 00000100:00000001:3.0:1787829317.479535:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298791680 00000100:00000001:3.0:1787829317.479536:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829317.479538:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829317.479539:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1787829317.479541:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829317/real 1787829317] req@ffff8d30d13eda40 x1874670646818048/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829342 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1787829317.479551:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787829317.479551:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829317.479553:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829317.479554:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829317.479554:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829317.479555:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829317.479556:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829317.479558:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d13eda40 x1874670646818048/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829342 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829317.479562:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829317.479564:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829317.479565:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829317.479566:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829317.479567:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829317.479568:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829317.479569:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829317.479571:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829317.479573:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829317.479574:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829317.479576:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d13eda40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646818048:192.168.203.143@tcp:8:kworker.0 00000100:00000001:3.0:1787829317.479578:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829317.479579:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829317.479579:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829317.479580:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829317.479581:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829317.479582:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829317.479583:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829317.479584:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829317.479585:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829317.479586:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829317.479587:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829317.479588:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829317.479590:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787829317.479591:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829317.479592:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1787829317.479593:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829318.374742:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829318.374747:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829318.374791:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829318.374795:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829318.374798:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829319.398671:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829319.398675:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829319.398730:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829319.398736:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829319.398741:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829320.422715:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829320.422723:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829320.422791:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829320.422797:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829320.422803:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829321.446734:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829321.446744:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829321.446818:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829321.446825:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829321.446830:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829322.470795:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829322.470808:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829322.470897:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829322.470903:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829322.470907:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787829322.598732:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829322.598746:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1787829322.598749:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787829322.598754:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787829322.598760:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787829322.598769:0:468:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 4123 00000100:00080000:2.0:1787829322.598783:0:468:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787829322.598788:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829322.598790:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787829322.598795:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787829322.598799:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787829322.598803:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:2.0:1787829322.598806:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:2.0:1787829322.598808:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829322.598810:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:2.0:1787829322.598816:0:468:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787829322.598819:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829322.598823:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787829322.598827:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787829322.598831:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829322.598834:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:2.0:1787829322.598837:0:468:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829322.598853:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787829322.598868:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829322.598873:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787829322.598876:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787829322.598881:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829322.598896:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787829322.598908:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2f480 x1874670646818176/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787829322.598921:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787829322.598926:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829322.598944:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787829322.598948:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829322.598952:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1787829322.598953:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787829322.598955:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787829322.598959:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787829322.598963:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4123 00000100:00080000:2.0:1787829322.598970:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787829322.598973:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829322.598976:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787829322.598979:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787829322.598980:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787829322.598984:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:3.0:1787829322.598986:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:2.0:1787829322.598988:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:2.0:1787829322.598993:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829322.598995:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1787829322.598996:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:3.0:1787829322.599001:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00080000:2.0:1787829322.599007:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:3.0:1787829322.599008:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787829322.599012:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829322.599013:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829322.599023:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787829322.599026:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787829322.599030:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829322.599032:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787829322.599039:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829322.599041:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829322.599044:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1787829322.599051:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1787829322.599054:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646818176:192.168.203.143@tcp:250:kworker.0 00000008:00000001:2.0:1787829322.599060:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829322.599071:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829322.599072:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787829322.599078:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787829322.599079:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1787829322.599085:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829322.599086:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829322.599096:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787829322.599097:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1787829322.599101:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787829322.599105:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829322.599106:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829322.599121:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829322.599128:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787829322.599130:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417881728 : -126234291669888 : ffff8d30c9e2f480) 00000100:00100000:2.0:1787829322.599136:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2da40 x1874670646818304/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787829322.599137:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:2.0:1787829322.599150:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787829322.599154:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829322.599158:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787829322.599162:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829322.599167:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1787829322.599169:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787829322.599172:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:3.0:1787829322.599174:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:2.0:1787829322.599181:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787829322.599187:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4123 00000100:00000001:3.0:1787829322.599190:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00080000:2.0:1787829322.599196:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787829322.599196:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829322.599202:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829322.599205:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829322.599206:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1787829322.599212:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:3.0:1787829322.599215:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1787829322.599218:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787829322.599222:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:3.0:1787829322.599222:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4298796800 00000020:00000001:2.0:1787829322.599230:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:3.0:1787829322.599235:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 00000020:00000001:2.0:1787829322.599238:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:2.0:1787829322.599243:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829322.599244:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4298796800 00000100:00000001:2.0:1787829322.599246:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:3.0:1787829322.599249:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 00000100:00000001:3.0:1787829322.599254:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00080000:2.0:1787829322.599260:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:3.0:1787829322.599260:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829322.599268:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00000001:2.0:1787829322.599270:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829322.599276:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00100000:3.0:1787829322.599276:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829322/real 1787829322] req@ffff8d30c9e2f480 x1874670646818176/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829347 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1787829322.599284:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000001:3.0:1787829322.599287:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 02000000:00000001:2.0:1787829322.599290:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:3.0:1787829322.599290:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829322.599295:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1787829322.599296:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:3.0:1787829322.599299:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829322.599302:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000008:00100000:2.0:1787829322.599303:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00000001:3.0:1787829322.599306:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829322.599310:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000008:00000001:2.0:1787829322.599311:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829322.599317:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2f480 x1874670646818176/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829347 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787829322.599319:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829322.599331:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:2.0:1787829322.599332:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:3.0:1787829322.599334:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 02000000:00000001:2.0:1787829322.599337:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829322.599338:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829322.599347:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787829322.599350:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:3.0:1787829322.599350:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:2.0:1787829322.599358:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:3.0:1787829322.599359:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829322.599364:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 02000000:00000001:2.0:1787829322.599366:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787829322.599368:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829322.599378:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829322.599380:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000001:2.0:1787829322.599382:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829322.599385:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646818176:192.168.203.143@tcp:250:kworker.0 00000100:00000001:3.0:1787829322.599390:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829322.599394:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00100000:2.0:1787829322.599395:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2c000 x1874670646818432/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787829322.599397:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829322.599400:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829322.599405:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829322.599408:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829322.599411:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829322.599416:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829322.599420:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1787829322.599423:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787829322.599424:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787829322.599429:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829322.599438:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829322.599439:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829322.599445:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:2.0:1787829322.599448:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829322.599454:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1787829322.599454:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787829322.599456:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829322.599459:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:3.0:1787829322.599460:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00080000:2.0:1787829322.599465:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787829322.599465:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787829322.599468:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1787829322.599471:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:3.0:1787829322.599471:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787829322.599477:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829322.599480:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000100:00080000:2.0:1787829322.599481:0:468:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4123 02000000:00000001:3.0:1787829322.599483:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829322.599488:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787829322.599493:0:468:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1787829322.599494:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2da40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646818304:192.168.203.143@tcp:38:kworker.0 00000100:00000001:2.0:1787829322.599502:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829322.599503:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1787829322.599508:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:2.0:1787829322.599509:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 02000000:00000001:3.0:1787829322.599512:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1787829322.599519:0:468:0:(genops.c:792:class_conn2export()) Process entered 02000000:00000001:3.0:1787829322.599520:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000020:00000001:2.0:1787829322.599527:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 02000000:00000001:3.0:1787829322.599528:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1787829322.599535:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:3.0:1787829322.599537:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787829322.599539:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417875008 : -126234291676608 : ffff8d30c9e2da40) 00000020:00000001:2.0:1787829322.599546:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:3.0:1787829322.599548:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:2.0:1787829322.599554:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829322.599557:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000800:00000100:3.0:1787829322.599562:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00080000:2.0:1787829322.599566:0:468:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:3.0:1787829322.599569:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787829322.599572:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829322.599573:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829322.599575:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1787829322.599576:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829322.599579:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298795542 00000100:00000001:2.0:1787829322.599586:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829322.599587:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298796800 00000100:00000001:3.0:1787829322.599591:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298795542 02000000:00000001:2.0:1787829322.599597:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:3.0:1787829322.599597:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298796800 00000100:00000001:3.0:1787829322.599629:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829322.599631:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829322.599632:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1787829322.599637:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829322/real 1787829322] req@ffff8d30c9e2da40 x1874670646818304/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829347 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1787829322.599644:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787829322.599645:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829322.599647:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829322.599648:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829322.599649:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829322.599651:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829322.599652:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829322.599656:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2da40 x1874670646818304/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829347 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1787829322.599659:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787829322.599669:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829322.599670:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829322.599678:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:3.0:1787829322.599678:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829322.599685:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000008:00100000:2.0:1787829322.599686:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00000001:3.0:1787829322.599687:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829322.599689:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000008:00000001:2.0:1787829322.599697:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829322.599697:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829322.599701:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787829322.599704:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787829322.599705:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829322.599715:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 02000000:00000001:2.0:1787829322.599719:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:3.0:1787829322.599720:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 02000000:00000001:2.0:1787829322.599726:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829322.599727:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2da40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646818304:192.168.203.143@tcp:38:kworker.0 00000100:00000001:2.0:1787829322.599736:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:3.0:1787829322.599736:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 02000000:00000001:2.0:1787829322.599745:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:3.0:1787829322.599745:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787829322.599751:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829322.599753:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829322.599753:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787829322.599757:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829322.599760:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829322.599762:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646818432:192.168.203.143@tcp:38:kworker.0 00000100:00000001:2.0:1787829322.599766:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829322.599767:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1787829322.599770:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1787829322.599775:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829322.599776:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1787829322.599779:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787829322.599784:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2e940 x1874670646818560/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787829322.599786:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787829322.599788:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417868288 : -126234291683328 : ffff8d30c9e2c000) 00000100:00000001:3.0:1787829322.599794:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:2.0:1787829322.599807:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000800:00000100:3.0:1787829322.599809:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:2.0:1787829322.599814:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829322.599814:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787829322.599818:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829322.599820:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829322.599822:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1787829322.599824:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829322.599826:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829322.599834:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298795543 00000100:00000001:3.0:1787829322.599837:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298796800 00000100:00000001:3.0:1787829322.599839:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298795543 00000100:00000001:3.0:1787829322.599841:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298796800 00000100:00000001:3.0:1787829322.599843:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829322.599845:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829322.599846:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1787829322.599849:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829322/real 1787829322] req@ffff8d30c9e2c000 x1874670646818432/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829347 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1787829322.599874:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787829322.599875:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829322.599878:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829322.599880:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829322.599881:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829322.599882:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829322.599883:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829322.599886:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2c000 x1874670646818432/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829347 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829322.599890:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829322.599892:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829322.599894:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829322.599895:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829322.599896:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829322.599897:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829322.599898:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829322.599899:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829322.599901:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829322.599902:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829322.599904:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646818432:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829322.599906:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829322.599908:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829322.599909:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829322.599910:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829322.599911:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829322.599913:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829322.599914:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829322.599915:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829322.599916:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829322.599918:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829322.599919:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829322.599921:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829322.599921:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829322.599922:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829322.599923:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829322.599924:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829322.599925:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829322.599926:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829322.599927:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829322.599928:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829322.599929:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829322.599930:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829322.599932:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787829322.599933:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829322.599934:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1787829322.599935:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787829322.599936:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787829322.599937:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787829322.599938:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829322.599940:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787829322.599941:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829322.599943:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829322.599945:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646818560:192.168.203.143@tcp:8:kworker.0 00000100:00000001:3.0:1787829322.599947:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1787829322.599949:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1787829322.599950:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829322.599951:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1787829322.599953:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829322.599956:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787829322.599957:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417878848 : -126234291672768 : ffff8d30c9e2e940) 00000100:00000001:3.0:1787829322.599959:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1787829322.599967:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:3.0:1787829322.599969:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787829322.599971:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829322.599972:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829322.599973:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1787829322.599974:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829322.599976:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298795556 00000100:00000001:3.0:1787829322.599978:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298796800 00000100:00000001:3.0:1787829322.599980:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298795556 00000100:00000001:3.0:1787829322.599981:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298796800 00000100:00000001:3.0:1787829322.599983:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829322.599984:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829322.599985:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1787829322.599987:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829322/real 1787829322] req@ffff8d30c9e2e940 x1874670646818560/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829347 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1787829322.599992:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787829322.599993:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829322.599994:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829322.599995:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829322.599996:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829322.599997:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829322.599997:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829322.600000:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2e940 x1874670646818560/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829347 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829322.600004:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829322.600005:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829322.600007:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829322.600008:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829322.600009:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829322.600010:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829322.600011:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829322.600012:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829322.600014:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829322.600015:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829322.600017:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646818560:192.168.203.143@tcp:8:kworker.0 00000100:00000001:3.0:1787829322.600019:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829322.600020:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829322.600021:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829322.600022:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829322.600024:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829322.600025:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829322.600025:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829322.600027:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829322.600027:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829322.600030:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829322.600031:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829322.600032:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829322.600033:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787829322.600034:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829322.600036:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1787829322.600037:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829323.494746:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829323.494756:0:815:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1787829323.494760:0:815:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1787829323.494764:0:815:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829323.494772:0:815:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1787829323.494776:0:815:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829323.494779:0:815:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829323.494783:0:815:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829323.494859:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829323.494865:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829323.494869:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829324.518781:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829324.518790:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829324.518883:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829324.518890:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829324.518896:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829325.542701:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829325.542708:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829325.542757:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829325.542761:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829325.542765:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829326.566734:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829326.566742:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829326.566800:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829326.566805:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829326.566808:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829327.590742:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829327.590749:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829327.590816:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829327.590832:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829327.590837:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787829327.718753:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829327.718765:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1787829327.718770:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787829327.718774:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787829327.718781:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787829327.718794:0:468:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 4128 00000100:00080000:2.0:1787829327.718809:0:468:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787829327.718814:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829327.718817:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787829327.718836:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787829327.718840:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787829327.718844:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:2.0:1787829327.718847:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:2.0:1787829327.718849:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829327.718851:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:2.0:1787829327.718858:0:468:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787829327.718861:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829327.718864:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787829327.718868:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787829327.718873:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829327.718876:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:2.0:1787829327.718879:0:468:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829327.718891:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787829327.718893:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829327.718897:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787829327.718900:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787829327.718906:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829327.718918:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787829327.718931:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2ed00 x1874670646818688/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787829327.718943:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787829327.718948:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829327.718969:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787829327.718973:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829327.718977:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1787829327.718979:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787829327.718982:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787829327.718987:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787829327.718992:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4128 00000100:00080000:2.0:1787829327.718997:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1787829327.718999:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787829327.719001:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829327.719005:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:1.0:1787829327.719008:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000020:00000001:2.0:1787829327.719010:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:1.0:1787829327.719012:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000020:00000001:2.0:1787829327.719015:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:1.0:1787829327.719017:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000020:00000001:2.0:1787829327.719021:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:1.0:1787829327.719022:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1787829327.719027:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 02000000:00000001:1.0:1787829327.719032:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:2.0:1787829327.719033:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829327.719037:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 02000000:00000001:1.0:1787829327.719039:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00080000:2.0:1787829327.719045:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 02000000:00000001:1.0:1787829327.719047:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829327.719052:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829327.719060:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646818688:192.168.203.143@tcp:250:kworker.0 02000000:00000001:2.0:1787829327.719061:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1787829327.719068:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787829327.719072:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829327.719073:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787829327.719077:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829327.719082:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787829327.719087:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829327.719091:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829327.719096:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:1.0:1787829327.719101:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000008:00100000:2.0:1787829327.719103:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00000001:1.0:1787829327.719104:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417879808 : -126234291671808 : ffff8d30c9e2ed00) 00000100:00000001:1.0:1787829327.719112:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000008:00000001:2.0:1787829327.719114:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829327.719119:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829327.719125:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787829327.719127:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829327.719129:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787829327.719131:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787829327.719134:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829327.719141:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:1.0:1787829327.719154:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00100000:2.0:1787829327.719155:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2f0c0 x1874670646818816/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787829327.719167:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787829327.719168:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787829327.719174:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787829327.719175:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829327.719178:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829327.719181:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1787829327.719185:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829327.719187:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787829327.719191:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1787829327.719192:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4298801920 00000100:00080000:2.0:1787829327.719201:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:1.0:1787829327.719205:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 00000100:00000001:2.0:1787829327.719205:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787829327.719213:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787829327.719216:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4298801920 00000100:00000001:2.0:1787829327.719225:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:1.0:1787829327.719227:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 00000100:00000001:1.0:1787829327.719232:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00080000:2.0:1787829327.719237:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4128 00000100:00000001:1.0:1787829327.719238:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829327.719246:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00080000:2.0:1787829327.719250:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1787829327.719256:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829327/real 1787829327] req@ffff8d30c9e2ed00 x1874670646818688/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829352 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787829327.719260:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829327.719264:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787829327.719274:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000400:00000001:1.0:1787829327.719277:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787829327.719279:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000020:00000001:2.0:1787829327.719279:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787829327.719287:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:1.0:1787829327.719288:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787829327.719292:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000020:00000001:2.0:1787829327.719294:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:1.0:1787829327.719296:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829327.719302:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829327.719305:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829327.719305:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829327.719315:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00100000:1.0:1787829327.719318:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2ed00 x1874670646818688/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829352 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787829327.719329:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829327.719334:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787829327.719338:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:1.0:1787829327.719339:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829327.719348:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787829327.719350:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829327.719352:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829327.719360:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 02000000:00000001:2.0:1787829327.719361:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1787829327.719364:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787829327.719370:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 02000000:00000001:2.0:1787829327.719370:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787829327.719382:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1787829327.719385:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:2.0:1787829327.719389:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:1.0:1787829327.719390:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000008:00100000:2.0:1787829327.719397:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829327.719399:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646818688:192.168.203.143@tcp:250:kworker.0 00000008:00000001:2.0:1787829327.719407:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829327.719414:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829327.719414:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829327.719424:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829327.719427:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787829327.719432:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1787829327.719432:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787829327.719437:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829327.719439:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787829327.719443:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000100:00000001:2.0:1787829327.719444:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000020:00000001:1.0:1787829327.719446:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:2.0:1787829327.719450:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1787829327.719452:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787829327.719456:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 02000000:00000001:2.0:1787829327.719458:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829327.719464:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787829327.719467:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829327.719469:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829327.719470:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829327.719488:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00100000:2.0:1787829327.719488:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2fc00 x1874670646818944/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787829327.719494:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829327.719506:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787829327.719510:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1787829327.719513:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1787829327.719513:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787829327.719521:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787829327.719525:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829327.719527:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829327.719537:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:2.0:1787829327.719539:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829327.719540:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829327.719552:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787829327.719552:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1787829327.719561:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646818816:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787829327.719574:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00080000:2.0:1787829327.719575:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 02000000:00000001:1.0:1787829327.719577:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:2.0:1787829327.719579:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 02000000:00000001:1.0:1787829327.719580:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829327.719584:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00080000:2.0:1787829327.719584:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787829327.719592:0:468:0:(import.c:485:import_select_connection()) Process entered 02000000:00000001:1.0:1787829327.719593:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787829327.719623:0:468:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4128 00000100:00000001:1.0:1787829327.719629:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00080000:2.0:1787829327.719629:0:468:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1787829327.719632:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417880768 : -126234291670848 : ffff8d30c9e2f0c0) 00000100:00000001:2.0:1787829327.719633:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829327.719642:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:1.0:1787829327.719644:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000020:00000001:2.0:1787829327.719651:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787829327.719655:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787829327.719659:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000020:00000001:2.0:1787829327.719661:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:2.0:1787829327.719664:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829327.719665:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:2.0:1787829327.719669:0:468:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000800:00000100:1.0:1787829327.719672:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:2.0:1787829327.719676:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829327.719680:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1787829327.719682:0:2390:0:(events.c:34:request_out_callback()) Process entered 02000000:00000001:2.0:1787829327.719684:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829327.719686:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 02000000:00000001:2.0:1787829327.719688:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829327.719689:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829327.719692:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:1.0:1787829327.719693:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000008:00100000:2.0:1787829327.719700:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00000001:1.0:1787829327.719702:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:2.0:1787829327.719711:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829327.719712:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298800663 00000100:00000001:2.0:1787829327.719715:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829327.719717:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298801920 02000000:00000001:2.0:1787829327.719723:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:1.0:1787829327.719724:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298800663 02000000:00000001:2.0:1787829327.719730:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829327.719732:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298801920 00000100:00000001:2.0:1787829327.719737:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:1.0:1787829327.719739:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787829327.719741:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:1.0:1787829327.719744:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829327.719746:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829327.719748:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00000001:2.0:1787829327.719753:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829327.719754:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829327/real 1787829327] req@ffff8d30c9e2f0c0 x1874670646818816/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829352 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829327.719767:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2de00 x1874670646819072/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000001:1.0:1787829327.719780:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787829327.719785:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787829327.719791:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1787829327.719792:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787829327.719800:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787829327.719801:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829327.719807:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829327.719808:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829327.719812:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829327.719814:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787829327.719834:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2f0c0 x1874670646818816/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829352 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787829327.719840:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829327.719843:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787829327.719847:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829327.719848:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829327.719849:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829327.719851:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787829327.719852:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787829327.719854:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829327.719857:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787829327.719858:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787829327.719861:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646818816:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787829327.719864:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829327.719866:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829327.719867:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787829327.719868:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787829327.719870:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787829327.719871:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787829327.719872:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787829327.719873:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787829327.719874:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787829327.719875:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787829327.719876:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829327.719878:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829327.719880:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787829327.719881:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829327.719883:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787829327.719885:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1787829327.719886:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787829327.719887:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787829327.719888:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829327.719890:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787829327.719892:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829327.719893:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829327.719896:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646818944:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787829327.719898:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787829327.719900:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787829327.719901:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829327.719902:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787829327.719905:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829327.719908:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787829327.719909:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417883648 : -126234291667968 : ffff8d30c9e2fc00) 00000100:00000001:1.0:1787829327.719912:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1787829327.719923:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787829327.719927:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787829327.719930:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829327.719931:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829327.719933:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787829327.719935:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829327.719939:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298800663 00000100:00000001:1.0:1787829327.719942:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298801920 00000100:00000001:1.0:1787829327.719944:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298800663 00000100:00000001:1.0:1787829327.719946:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298801920 00000100:00000001:1.0:1787829327.719949:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829327.719951:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829327.719953:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787829327.719958:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829327/real 1787829327] req@ffff8d30c9e2fc00 x1874670646818944/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829352 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787829327.719968:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787829327.719970:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787829327.719972:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787829327.719974:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787829327.719975:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829327.719976:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829327.719977:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787829327.719980:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2fc00 x1874670646818944/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829352 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787829327.719985:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829327.719986:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787829327.719988:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829327.719989:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829327.719990:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829327.719991:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787829327.719992:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787829327.719993:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829327.719995:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787829327.719996:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787829327.719998:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646818944:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787829327.720001:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787829327.720002:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787829327.720003:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829327.720006:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787829327.720008:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829327.720010:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829327.720013:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646819072:192.168.203.143@tcp:8:kworker.0 00000100:00000001:1.0:1787829327.720016:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787829327.720018:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787829327.720020:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829327.720022:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787829327.720025:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829327.720031:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787829327.720032:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417875968 : -126234291675648 : ffff8d30c9e2de00) 00000100:00000001:1.0:1787829327.720036:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1787829327.720049:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787829327.720053:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787829327.720055:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829327.720056:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829327.720057:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787829327.720059:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829327.720061:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298800676 00000100:00000001:1.0:1787829327.720063:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298801920 00000100:00000001:1.0:1787829327.720064:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298800676 00000100:00000001:1.0:1787829327.720066:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298801920 00000100:00000001:1.0:1787829327.720067:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829327.720069:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829327.720070:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787829327.720073:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829327/real 1787829327] req@ffff8d30c9e2de00 x1874670646819072/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829352 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787829327.720078:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787829327.720078:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787829327.720080:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787829327.720081:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787829327.720082:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829327.720083:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829327.720084:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787829327.720086:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2de00 x1874670646819072/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829352 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787829327.720091:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829327.720092:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787829327.720094:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829327.720095:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829327.720096:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829327.720097:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787829327.720098:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787829327.720100:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829327.720102:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787829327.720103:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787829327.720105:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646819072:192.168.203.143@tcp:8:kworker.0 00000100:00000001:1.0:1787829327.720107:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829327.720109:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829327.720109:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787829327.720110:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787829327.720112:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787829327.720113:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787829327.720114:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787829327.720115:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787829327.720116:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787829327.720118:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787829327.720118:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829327.720120:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829327.720120:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787829327.720121:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787829327.720122:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787829327.720123:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787829327.720124:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787829327.720125:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787829327.720126:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787829327.720128:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787829327.720129:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829327.720130:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829327.720132:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787829327.720133:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829327.720134:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787829327.720135:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829328.614716:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829328.614723:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829328.614785:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829328.614790:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829328.614794:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829329.638764:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829329.638771:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829329.638850:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829329.638856:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829329.638862:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829330.662743:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829330.662751:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829330.662834:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829330.662838:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829330.662842:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829331.686735:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829331.686743:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829331.686826:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829331.686834:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829331.686838:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829332.710732:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829332.710740:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829332.710803:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829332.710806:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829332.710809:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787829332.838726:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829332.838734:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1787829332.838736:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787829332.838739:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787829332.838743:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787829332.838749:0:468:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 4133 00000100:00080000:0.0:1787829332.838758:0:468:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787829332.838760:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829332.838762:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787829332.838765:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787829332.838767:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787829332.838770:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:0.0:1787829332.838772:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:0.0:1787829332.838773:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829332.838774:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:0.0:1787829332.838777:0:468:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787829332.838779:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829332.838781:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787829332.838797:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787829332.838801:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829332.838803:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1787829332.838805:0:468:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829332.838814:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787829332.838815:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829332.838818:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787829332.838821:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787829332.838824:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829332.838833:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787829332.838840:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d27c00 x1874670646819200/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787829332.838848:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787829332.838851:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829332.838870:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787829332.838872:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829332.838874:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1787829332.838876:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787829332.838877:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787829332.838879:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787829332.838881:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4133 00000100:00080000:0.0:1787829332.838884:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787829332.838886:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829332.838887:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787829332.838889:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787829332.838891:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787829332.838893:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000020:00000001:0.0:1787829332.838895:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:0.0:1787829332.838896:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829332.838899:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:0.0:1787829332.838901:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787829332.838903:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829332.838904:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787829332.838908:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787829332.838910:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829332.838914:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:1.0:1787829332.838915:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000008:00100000:0.0:1787829332.838921:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1787829332.838924:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829332.838926:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829332.838926:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 02000000:00000001:0.0:1787829332.838932:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787829332.838934:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829332.838934:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1787829332.838940:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787829332.838941:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:1.0:1787829332.838941:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787829332.838948:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829332.838949:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829332.838960:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829332.838964:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000100:00100000:0.0:1787829332.838965:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d252c0 x1874670646819328/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:1.0:1787829332.838970:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787829332.838974:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 02000000:00000001:1.0:1787829332.838978:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829332.838979:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829332.838996:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787829332.838999:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829332.839004:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:1.0:1787829332.839005:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d27c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646819200:192.168.203.143@tcp:250:kworker.0 00000100:00000001:0.0:1787829332.839006:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787829332.839008:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787829332.839014:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787829332.839021:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4133 00000100:00000001:1.0:1787829332.839022:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00080000:0.0:1787829332.839025:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787829332.839027:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829332.839028:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787829332.839029:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787829332.839030:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 02000000:00000001:1.0:1787829332.839031:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000020:00000001:0.0:1787829332.839032:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000020:00000001:0.0:1787829332.839037:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 02000000:00000001:1.0:1787829332.839038:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829332.839039:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829332.839042:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:0.0:1787829332.839048:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 02000000:00000001:1.0:1787829332.839048:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:0.0:1787829332.839053:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829332.839054:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787829332.839056:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787829332.839057:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829332.839059:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 02000000:00000001:1.0:1787829332.839060:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000008:00100000:0.0:1787829332.839061:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1787829332.839069:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829332.839071:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829332.839073:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787829332.839074:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829332.839075:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787829332.839077:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787829332.839079:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829332.839084:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787829332.839088:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d25680 x1874670646819456/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787829332.839091:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787829332.839093:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787829332.839098:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601351680 : -126234108199936 : ffff8d30d4d27c00) 00000100:00000001:0.0:1787829332.839099:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829332.839104:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787829332.839109:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1787829332.839111:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00080000:0.0:1787829332.839112:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1787829332.839117:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787829332.839118:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787829332.839120:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787829332.839122:0:468:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4133 00000100:00080000:0.0:1787829332.839125:0:468:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787829332.839127:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829332.839128:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787829332.839129:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787829332.839130:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787829332.839131:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000020:00000001:0.0:1787829332.839134:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:0.0:1787829332.839136:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829332.839137:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:0.0:1787829332.839140:0:468:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787829332.839141:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829332.839142:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787829332.839144:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787829332.839145:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829332.839146:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1787829332.839148:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1787829332.839150:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829332.839152:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829332.839155:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787829332.839156:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829332.839157:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787829332.839157:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787829332.839159:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829332.839162:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787829332.839173:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d26940 x1874670646819584/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787829332.839177:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:0.0:1787829332.839190:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787829332.839193:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829332.839196:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829332.839200:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787829332.839209:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829332.839213:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829332.839215:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787829332.839225:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829332.839230:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4298807040 00000100:00000001:1.0:1787829332.839244:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 00000100:00000001:1.0:1787829332.839252:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4298807040 00000100:00000001:1.0:1787829332.839257:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 00000100:00000001:1.0:1787829332.839261:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829332.839264:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829332.839267:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787829332.839276:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829332/real 1787829332] req@ffff8d30d4d27c00 x1874670646819200/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829357 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787829332.839288:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787829332.839290:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787829332.839294:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787829332.839297:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787829332.839300:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829332.839303:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829332.839315:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787829332.839321:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d27c00 x1874670646819200/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829357 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787829332.839330:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829332.839334:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787829332.839338:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829332.839340:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829332.839341:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829332.839344:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787829332.839347:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787829332.839351:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829332.839356:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787829332.839359:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787829332.839363:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d27c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646819200:192.168.203.143@tcp:250:kworker.0 00000100:00000001:1.0:1787829332.839370:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829332.839372:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829332.839374:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787829332.839377:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787829332.839382:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787829332.839385:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787829332.839388:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787829332.839393:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787829332.839395:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787829332.839400:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787829332.839402:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829332.839405:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829332.839415:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787829332.839418:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829332.839421:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787829332.839423:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1787829332.839424:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787829332.839426:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787829332.839428:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829332.839431:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787829332.839433:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829332.839436:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829332.839440:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d252c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646819328:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787829332.839444:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787829332.839446:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787829332.839448:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829332.839450:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787829332.839453:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829332.839460:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787829332.839462:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601341120 : -126234108210496 : ffff8d30d4d252c0) 00000100:00000001:1.0:1787829332.839467:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1787829332.839478:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787829332.839482:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787829332.839484:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829332.839485:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829332.839486:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787829332.839489:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829332.839492:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298805782 00000100:00000001:1.0:1787829332.839494:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298807040 00000100:00000001:1.0:1787829332.839496:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298805782 00000100:00000001:1.0:1787829332.839498:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298807040 00000100:00000001:1.0:1787829332.839499:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829332.839501:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829332.839502:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787829332.839505:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829332/real 1787829332] req@ffff8d30d4d252c0 x1874670646819328/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829357 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787829332.839510:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787829332.839511:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787829332.839513:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787829332.839514:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787829332.839515:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829332.839516:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829332.839518:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787829332.839520:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d252c0 x1874670646819328/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829357 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787829332.839533:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829332.839534:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787829332.839536:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829332.839537:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829332.839538:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829332.839540:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787829332.839541:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787829332.839543:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829332.839546:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787829332.839547:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787829332.839549:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d252c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646819328:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787829332.839552:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787829332.839554:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787829332.839555:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829332.839556:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787829332.839557:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829332.839559:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829332.839561:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d25680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646819456:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787829332.839564:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787829332.839565:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787829332.839566:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829332.839568:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787829332.839570:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829332.839573:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787829332.839574:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601342080 : -126234108209536 : ffff8d30d4d25680) 00000100:00000001:1.0:1787829332.839576:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1787829332.839583:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787829332.839586:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787829332.839588:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829332.839589:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829332.839590:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787829332.839592:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829332.839595:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298805782 00000100:00000001:1.0:1787829332.839597:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298807040 00000100:00000001:1.0:1787829332.839598:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298805782 00000100:00000001:1.0:1787829332.839622:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298807040 00000100:00000001:1.0:1787829332.839624:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829332.839626:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829332.839627:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787829332.839630:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829332/real 1787829332] req@ffff8d30d4d25680 x1874670646819456/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829357 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787829332.839635:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787829332.839636:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787829332.839638:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787829332.839639:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787829332.839640:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829332.839641:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829332.839642:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787829332.839644:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d25680 x1874670646819456/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829357 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787829332.839648:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829332.839650:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787829332.839651:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829332.839652:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829332.839653:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829332.839654:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787829332.839655:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787829332.839657:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829332.839659:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787829332.839661:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787829332.839663:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d25680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646819456:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787829332.839666:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787829332.839667:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787829332.839668:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829332.839670:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787829332.839671:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829332.839672:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829332.839674:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d26940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646819584:192.168.203.143@tcp:8:kworker.0 00000100:00000001:1.0:1787829332.839675:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787829332.839677:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787829332.839678:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829332.839679:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787829332.839682:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829332.839685:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787829332.839686:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601346880 : -126234108204736 : ffff8d30d4d26940) 00000100:00000001:1.0:1787829332.839689:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1787829332.839696:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787829332.839698:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787829332.839700:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829332.839701:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829332.839702:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787829332.839704:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829332.839706:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298805796 00000100:00000001:1.0:1787829332.839708:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298807040 00000100:00000001:1.0:1787829332.839710:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298805796 00000100:00000001:1.0:1787829332.839711:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298807040 00000100:00000001:1.0:1787829332.839713:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829332.839714:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829332.839715:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787829332.839718:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829332/real 1787829332] req@ffff8d30d4d26940 x1874670646819584/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829357 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787829332.839728:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787829332.839729:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787829332.839730:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787829332.839731:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787829332.839732:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829332.839733:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829332.839734:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787829332.839736:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d26940 x1874670646819584/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829357 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787829332.839740:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829332.839741:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787829332.839743:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829332.839744:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829332.839745:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829332.839746:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787829332.839748:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787829332.839749:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829332.839751:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787829332.839752:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787829332.839754:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d26940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646819584:192.168.203.143@tcp:8:kworker.0 00000100:00000001:1.0:1787829332.839756:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829332.839758:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829332.839758:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787829332.839759:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787829332.839761:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787829332.839762:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787829332.839762:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787829332.839764:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787829332.839765:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787829332.839768:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787829332.839768:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829332.839770:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829332.839771:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787829332.839772:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787829332.839774:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787829332.839775:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787829332.839776:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787829332.839777:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787829332.839778:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787829332.839779:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787829332.839779:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829332.839780:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829332.839781:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787829332.839782:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787829332.839796:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787829332.839797:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787829332.839798:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787829332.839799:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787829332.839799:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787829332.839801:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787829332.839802:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829332.839804:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829332.839806:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787829332.839807:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829332.839808:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787829332.839809:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829333.734727:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829333.734736:0:815:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1787829333.734738:0:815:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1787829333.734740:0:815:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829333.734746:0:815:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1787829333.734748:0:815:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829333.734749:0:815:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829333.734751:0:815:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829333.734914:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829333.734917:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829333.734920:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829334.758683:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829334.758690:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829334.758733:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829334.758736:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829334.758739:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829335.782701:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829335.782708:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829335.782777:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829335.782780:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829335.782784:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829336.806678:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829336.806683:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829336.806739:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829336.806744:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829336.806747:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829337.830671:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829337.830677:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829337.830727:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829337.830732:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829337.830735:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787829337.958710:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829337.958720:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1787829337.958723:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787829337.958727:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787829337.958732:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787829337.958740:0:468:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 4138 00000100:00080000:2.0:1787829337.958762:0:468:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787829337.958766:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829337.958769:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787829337.958773:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787829337.958775:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787829337.958779:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:2.0:1787829337.958782:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:2.0:1787829337.958784:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829337.958786:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:2.0:1787829337.958790:0:468:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787829337.958793:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829337.958797:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787829337.958800:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787829337.958804:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829337.958806:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:2.0:1787829337.958808:0:468:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829337.958820:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787829337.958822:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829337.958826:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787829337.958829:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787829337.958833:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829337.958845:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787829337.958855:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2c3c0 x1874670646819712/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787829337.958866:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787829337.958871:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829337.958886:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787829337.958888:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829337.958891:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1787829337.958892:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787829337.958894:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787829337.958896:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787829337.958897:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4138 00000100:00080000:2.0:1787829337.958900:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787829337.958902:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829337.958903:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787829337.958905:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787829337.958906:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787829337.958908:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000020:00000001:2.0:1787829337.958909:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:2.0:1787829337.958911:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829337.958912:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:2.0:1787829337.958914:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787829337.958916:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829337.958917:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787829337.958922:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787829337.958923:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829337.958925:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1787829337.958928:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1787829337.958930:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829337.958932:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787829337.958932:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829337.958938:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 02000000:00000001:2.0:1787829337.958939:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:1.0:1787829337.958941:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 02000000:00000001:2.0:1787829337.958942:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829337.958946:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:1.0:1787829337.958947:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 02000000:00000001:2.0:1787829337.958948:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787829337.958951:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829337.958953:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829337.958956:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829337.958959:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000100:00100000:2.0:1787829337.958962:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2c780 x1874670646819840/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:1.0:1787829337.958965:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829337.958971:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829337.958975:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787829337.958977:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829337.958982:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646819712:192.168.203.143@tcp:250:kworker.0 00000100:00000001:2.0:1787829337.958982:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787829337.958984:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829337.958990:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1787829337.958992:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:1.0:1787829337.958993:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00080000:2.0:1787829337.958994:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787829337.958998:0:468:0:(import.c:485:import_select_connection()) Process entered 02000000:00000001:1.0:1787829337.959001:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000100:00080000:2.0:1787829337.959001:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4138 02000000:00000001:1.0:1787829337.959006:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787829337.959008:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 02000000:00000001:1.0:1787829337.959010:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:2.0:1787829337.959013:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829337.959016:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 02000000:00000001:1.0:1787829337.959017:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1787829337.959021:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787829337.959024:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787829337.959026:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000020:00000001:2.0:1787829337.959029:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:2.0:1787829337.959032:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829337.959033:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:1.0:1787829337.959036:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00080000:2.0:1787829337.959038:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:1.0:1787829337.959040:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417869248 : -126234291682368 : ffff8d30c9e2c3c0) 00000100:00000001:2.0:1787829337.959044:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829337.959047:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 02000000:00000001:2.0:1787829337.959048:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787829337.959053:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787829337.959056:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829337.959058:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1787829337.959061:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1787829337.959064:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829337.959066:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829337.959071:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787829337.959073:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829337.959075:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787829337.959077:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787829337.959079:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829337.959085:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:1.0:1787829337.959088:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00100000:2.0:1787829337.959094:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2d2c0 x1874670646819968/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787829337.959099:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787829337.959104:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787829337.959106:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787829337.959107:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829337.959110:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1787829337.959111:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829337.959115:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829337.959117:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829337.959120:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4298812160 00000100:00080000:2.0:1787829337.959122:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829337.959127:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1787829337.959129:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 00000100:00000001:2.0:1787829337.959130:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787829337.959134:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787829337.959136:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4298812160 00000100:00000001:2.0:1787829337.959140:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:1.0:1787829337.959141:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 00000100:00000001:1.0:1787829337.959144:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829337.959146:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829337.959148:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787829337.959153:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829337/real 1787829337] req@ffff8d30c9e2c3c0 x1874670646819712/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829362 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829337.959154:0:468:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4138 00000100:00080000:2.0:1787829337.959162:0:468:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000400:00000001:1.0:1787829337.959165:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000001:2.0:1787829337.959167:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000001:1.0:1787829337.959168:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1787829337.959172:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:1.0:1787829337.959173:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787829337.959176:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000020:00000001:2.0:1787829337.959177:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:1.0:1787829337.959178:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:2.0:1787829337.959180:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:1.0:1787829337.959181:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829337.959184:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:2.0:1787829337.959184:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000020:00000001:2.0:1787829337.959190:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00100000:1.0:1787829337.959192:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2c3c0 x1874670646819712/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829362 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787829337.959195:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829337.959199:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:1.0:1787829337.959203:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829337.959208:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787829337.959208:0:468:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:1.0:1787829337.959214:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829337.959214:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829337.959219:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1787829337.959220:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829337.959223:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:2.0:1787829337.959224:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829337.959227:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 02000000:00000001:2.0:1787829337.959227:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829337.959230:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:1.0:1787829337.959231:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000008:00100000:2.0:1787829337.959233:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000008:00000001:2.0:1787829337.959236:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829337.959240:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1787829337.959241:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 02000000:00000001:2.0:1787829337.959244:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787829337.959247:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1787829337.959249:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:2.0:1787829337.959250:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787829337.959252:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787829337.959256:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829337.959257:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000001:2.0:1787829337.959260:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829337.959265:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646819712:192.168.203.143@tcp:250:kworker.0 00000100:00100000:2.0:1787829337.959268:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2cb40 x1874670646820096/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787829337.959277:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829337.959278:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787829337.959282:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829337.959283:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787829337.959285:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829337.959287:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787829337.959291:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787829337.959296:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787829337.959298:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787829337.959300:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787829337.959305:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787829337.959307:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787829337.959310:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787829337.959311:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829337.959313:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829337.959318:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787829337.959319:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829337.959321:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787829337.959322:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1787829337.959323:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787829337.959324:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787829337.959325:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829337.959327:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787829337.959329:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829337.959330:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829337.959333:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646819840:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787829337.959336:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787829337.959337:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787829337.959339:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829337.959340:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787829337.959342:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829337.959347:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787829337.959348:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417870208 : -126234291681408 : ffff8d30c9e2c780) 00000100:00000001:1.0:1787829337.959351:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1787829337.959362:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787829337.959366:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787829337.959367:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829337.959368:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829337.959369:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787829337.959371:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829337.959374:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298810902 00000100:00000001:1.0:1787829337.959377:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298812160 00000100:00000001:1.0:1787829337.959378:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298810902 00000100:00000001:1.0:1787829337.959380:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298812160 00000100:00000001:1.0:1787829337.959381:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829337.959383:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829337.959384:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787829337.959386:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829337/real 1787829337] req@ffff8d30c9e2c780 x1874670646819840/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829362 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787829337.959392:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787829337.959392:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787829337.959394:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787829337.959395:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787829337.959396:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829337.959397:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829337.959398:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787829337.959401:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2c780 x1874670646819840/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829362 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787829337.959405:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829337.959406:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787829337.959408:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829337.959409:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829337.959410:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829337.959411:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787829337.959412:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787829337.959413:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829337.959416:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787829337.959417:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787829337.959420:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646819840:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787829337.959422:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787829337.959423:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787829337.959424:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829337.959425:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787829337.959426:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829337.959427:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829337.959429:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2d2c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646819968:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787829337.959430:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787829337.959432:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787829337.959433:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829337.959434:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787829337.959436:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829337.959439:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787829337.959440:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417873088 : -126234291678528 : ffff8d30c9e2d2c0) 00000100:00000001:1.0:1787829337.959442:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1787829337.959448:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787829337.959450:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787829337.959452:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829337.959452:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829337.959454:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787829337.959455:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829337.959457:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298810902 00000100:00000001:1.0:1787829337.959459:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298812160 00000100:00000001:1.0:1787829337.959460:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298810902 00000100:00000001:1.0:1787829337.959462:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298812160 00000100:00000001:1.0:1787829337.959464:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829337.959465:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829337.959466:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787829337.959468:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829337/real 1787829337] req@ffff8d30c9e2d2c0 x1874670646819968/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829362 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787829337.959473:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787829337.959474:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787829337.959476:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787829337.959476:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787829337.959477:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829337.959478:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829337.959479:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787829337.959481:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2d2c0 x1874670646819968/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829362 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787829337.959486:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829337.959487:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787829337.959489:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829337.959490:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829337.959490:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829337.959491:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787829337.959492:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787829337.959494:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829337.959495:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787829337.959496:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787829337.959498:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2d2c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646819968:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787829337.959500:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787829337.959501:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787829337.959502:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829337.959503:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787829337.959504:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829337.959505:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829337.959507:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646820096:192.168.203.143@tcp:8:kworker.0 00000100:00000001:1.0:1787829337.959509:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787829337.959510:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787829337.959511:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829337.959512:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787829337.959514:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829337.959517:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787829337.959518:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417871168 : -126234291680448 : ffff8d30c9e2cb40) 00000100:00000001:1.0:1787829337.959520:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1787829337.959525:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787829337.959527:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787829337.959528:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829337.959529:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829337.959531:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787829337.959532:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829337.959534:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298810915 00000100:00000001:1.0:1787829337.959536:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298812160 00000100:00000001:1.0:1787829337.959537:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298810915 00000100:00000001:1.0:1787829337.959538:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298812160 00000100:00000001:1.0:1787829337.959540:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829337.959541:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829337.959542:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787829337.959544:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829337/real 1787829337] req@ffff8d30c9e2cb40 x1874670646820096/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829362 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787829337.959549:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787829337.959549:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787829337.959550:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787829337.959551:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787829337.959552:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829337.959553:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829337.959554:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787829337.959556:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2cb40 x1874670646820096/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829362 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787829337.959560:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829337.959561:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787829337.959563:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829337.959564:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829337.959565:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829337.959566:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787829337.959567:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787829337.959568:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829337.959570:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787829337.959571:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787829337.959573:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646820096:192.168.203.143@tcp:8:kworker.0 00000100:00000001:1.0:1787829337.959575:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829337.959576:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829337.959577:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787829337.959578:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787829337.959579:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787829337.959580:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787829337.959581:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787829337.959582:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787829337.959583:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787829337.959584:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787829337.959585:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829337.959586:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829337.959586:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787829337.959587:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787829337.959588:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787829337.959589:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787829337.959590:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787829337.959591:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787829337.959591:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787829337.959594:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787829337.959595:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829337.959596:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829337.959596:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787829337.959597:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787829337.959598:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787829337.959599:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787829337.959625:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787829337.959626:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787829337.959628:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787829337.959629:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787829337.959630:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829337.959631:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829337.959633:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787829337.959634:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829337.959636:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787829337.959637:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829338.854713:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829338.854718:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829338.854781:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829338.854796:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829338.854799:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829339.878745:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829339.878751:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829339.878813:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829339.878816:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829339.878819:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829340.902688:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829340.902694:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829340.902761:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829340.902765:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829340.902768:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829341.926707:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829341.926712:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829341.926774:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829341.926777:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829341.926780:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829342.950688:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829342.950693:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829342.950757:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829342.950760:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829342.950762:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787829343.078733:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829343.078744:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1787829343.078747:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787829343.078752:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787829343.078756:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787829343.078764:0:468:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 4143 00000100:00080000:0.0:1787829343.078787:0:468:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787829343.078794:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829343.078796:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787829343.078801:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787829343.078804:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787829343.078807:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:0.0:1787829343.078810:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:0.0:1787829343.078812:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829343.078813:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:0.0:1787829343.078819:0:468:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787829343.078821:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829343.078825:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787829343.078828:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787829343.078832:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829343.078834:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1787829343.078837:0:468:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829343.078849:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787829343.078851:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829343.078855:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787829343.078857:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787829343.078861:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829343.078873:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787829343.078883:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d26940 x1874670646820224/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787829343.078894:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787829343.078898:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829343.078915:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787829343.078919:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829343.078922:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1787829343.078924:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787829343.078926:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787829343.078929:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787829343.078931:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4143 00000100:00080000:0.0:1787829343.078934:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787829343.078936:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829343.078937:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787829343.078939:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787829343.078940:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787829343.078942:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000020:00000001:0.0:1787829343.078944:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:0.0:1787829343.078945:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829343.078946:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:0.0:1787829343.078949:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787829343.078951:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829343.078953:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787829343.078957:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787829343.078959:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829343.078960:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787829343.078961:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:1.0:1787829343.078966:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000008:00100000:0.0:1787829343.078967:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1787829343.078970:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829343.078970:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787829343.078974:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787829343.078975:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829343.078978:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829343.078980:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787829343.078983:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829343.078984:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1787829343.078985:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787829343.078986:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787829343.078991:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829343.078992:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787829343.078995:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829343.078997:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787829343.079002:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d25680 x1874670646820352/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787829343.079007:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d26940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646820224:192.168.203.143@tcp:250:kworker.0 00000100:00000001:1.0:1787829343.079019:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:0.0:1787829343.079023:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787829343.079025:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829343.079026:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:0.0:1787829343.079028:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829343.079031:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787829343.079032:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829343.079037:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 02000000:00000001:1.0:1787829343.079038:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:0.0:1787829343.079039:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787829343.079041:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787829343.079043:0:468:0:(import.c:485:import_select_connection()) Process entered 02000000:00000001:1.0:1787829343.079044:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787829343.079046:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4143 00000100:00080000:0.0:1787829343.079049:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787829343.079051:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829343.079052:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787829343.079054:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787829343.079054:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787829343.079056:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000020:00000001:0.0:1787829343.079057:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:0.0:1787829343.079059:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829343.079060:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:1.0:1787829343.079060:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00080000:0.0:1787829343.079063:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:1.0:1787829343.079064:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601346880 : -126234108204736 : ffff8d30d4d26940) 00000100:00000001:0.0:1787829343.079067:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829343.079069:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 02000000:00000001:0.0:1787829343.079070:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787829343.079074:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787829343.079076:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829343.079077:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1787829343.079078:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1787829343.079080:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829343.079081:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829343.079085:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787829343.079086:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829343.079088:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787829343.079089:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787829343.079090:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829343.079092:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787829343.079095:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d252c0 x1874670646820480/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787829343.079101:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787829343.079103:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829343.079104:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787829343.079106:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829343.079108:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1787829343.079111:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787829343.079113:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:1.0:1787829343.079114:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:0.0:1787829343.079116:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787829343.079121:0:468:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4143 00000100:00080000:0.0:1787829343.079124:0:468:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787829343.079125:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829343.079127:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787829343.079128:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787829343.079129:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787829343.079131:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:1.0:1787829343.079131:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000020:00000001:0.0:1787829343.079136:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:0.0:1787829343.079137:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829343.079138:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:1.0:1787829343.079139:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00080000:0.0:1787829343.079142:0:468:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:1.0:1787829343.079144:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829343.079145:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829343.079150:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1787829343.079151:0:2390:0:(events.c:68:request_out_callback()) Process leaving 02000000:00000001:0.0:1787829343.079153:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787829343.079158:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829343.079160:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:1.0:1787829343.079160:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829343.079167:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4298817280 00000008:00100000:0.0:1787829343.079172:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00000001:1.0:1787829343.079178:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 00000008:00000001:0.0:1787829343.079181:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829343.079185:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829343.079188:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4298817280 02000000:00000001:0.0:1787829343.079190:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787829343.079193:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829343.079196:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:1.0:1787829343.079196:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 02000000:00000001:0.0:1787829343.079201:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:1.0:1787829343.079203:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787829343.079204:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829343.079211:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829343.079213:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829343.079214:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787829343.079220:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d27c00 x1874670646820608/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787829343.079223:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829343/real 1787829343] req@ffff8d30d4d26940 x1874670646820224/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829368 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787829343.079243:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000400:00000001:1.0:1787829343.079246:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000001:0.0:1787829343.079247:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:1.0:1787829343.079249:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787829343.079254:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829343.079255:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787829343.079260:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787829343.079262:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829343.079264:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829343.079266:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787829343.079270:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d26940 x1874670646820224/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829368 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787829343.079276:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829343.079279:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787829343.079281:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829343.079282:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829343.079292:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829343.079293:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787829343.079295:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787829343.079297:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829343.079301:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787829343.079303:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787829343.079305:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d26940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646820224:192.168.203.143@tcp:250:kworker.0 00000100:00000001:1.0:1787829343.079309:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829343.079311:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829343.079312:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787829343.079314:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787829343.079317:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787829343.079318:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787829343.079320:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787829343.079323:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787829343.079325:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787829343.079327:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787829343.079328:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829343.079330:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829343.079335:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787829343.079336:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829343.079338:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787829343.079339:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1787829343.079343:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787829343.079344:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787829343.079346:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829343.079347:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787829343.079349:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829343.079350:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829343.079352:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d25680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646820352:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787829343.079354:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787829343.079356:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787829343.079357:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829343.079359:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787829343.079361:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829343.079365:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787829343.079366:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601342080 : -126234108209536 : ffff8d30d4d25680) 00000100:00000001:1.0:1787829343.079370:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1787829343.079380:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787829343.079383:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787829343.079385:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829343.079386:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829343.079387:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787829343.079389:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829343.079392:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298816022 00000100:00000001:1.0:1787829343.079395:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298817280 00000100:00000001:1.0:1787829343.079396:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298816022 00000100:00000001:1.0:1787829343.079398:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298817280 00000100:00000001:1.0:1787829343.079399:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829343.079401:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829343.079402:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787829343.079405:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829343/real 1787829343] req@ffff8d30d4d25680 x1874670646820352/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829368 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787829343.079411:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787829343.079412:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787829343.079413:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787829343.079415:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787829343.079416:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829343.079417:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829343.079418:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787829343.079420:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d25680 x1874670646820352/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829368 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787829343.079425:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829343.079427:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787829343.079428:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829343.079429:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829343.079430:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829343.079431:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787829343.079432:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787829343.079434:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829343.079436:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787829343.079437:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787829343.079439:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d25680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646820352:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787829343.079442:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787829343.079442:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787829343.079443:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829343.079445:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787829343.079446:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829343.079447:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829343.079449:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d252c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646820480:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787829343.079452:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787829343.079454:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787829343.079455:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829343.079456:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787829343.079458:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829343.079461:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787829343.079462:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601341120 : -126234108210496 : ffff8d30d4d252c0) 00000100:00000001:1.0:1787829343.079464:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1787829343.079471:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787829343.079473:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787829343.079475:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829343.079476:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829343.079477:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787829343.079478:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829343.079480:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298816022 00000100:00000001:1.0:1787829343.079482:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298817280 00000100:00000001:1.0:1787829343.079484:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298816022 00000100:00000001:1.0:1787829343.079485:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298817280 00000100:00000001:1.0:1787829343.079487:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829343.079488:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829343.079489:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787829343.079491:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829343/real 1787829343] req@ffff8d30d4d252c0 x1874670646820480/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829368 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787829343.079496:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787829343.079497:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787829343.079498:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787829343.079499:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787829343.079500:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829343.079501:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829343.079502:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787829343.079504:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d252c0 x1874670646820480/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829368 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787829343.079514:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829343.079516:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787829343.079518:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829343.079520:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829343.079521:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829343.079523:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787829343.079524:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787829343.079527:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829343.079530:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787829343.079533:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787829343.079536:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d252c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646820480:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787829343.079540:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787829343.079542:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787829343.079543:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829343.079545:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787829343.079547:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829343.079549:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829343.079552:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d27c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646820608:192.168.203.143@tcp:8:kworker.0 00000100:00000001:1.0:1787829343.079555:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787829343.079558:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787829343.079559:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829343.079562:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787829343.079565:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829343.079571:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787829343.079572:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601351680 : -126234108199936 : ffff8d30d4d27c00) 00000100:00000001:1.0:1787829343.079576:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1787829343.079588:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787829343.079593:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787829343.079595:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829343.079597:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829343.079636:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787829343.079641:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829343.079644:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298816036 00000100:00000001:1.0:1787829343.079647:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298817280 00000100:00000001:1.0:1787829343.079650:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298816036 00000100:00000001:1.0:1787829343.079651:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298817280 00000100:00000001:1.0:1787829343.079653:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829343.079655:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829343.079656:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787829343.079660:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829343/real 1787829343] req@ffff8d30d4d27c00 x1874670646820608/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829368 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787829343.079666:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787829343.079667:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787829343.079669:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787829343.079670:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787829343.079671:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829343.079672:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829343.079673:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787829343.079676:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d27c00 x1874670646820608/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829368 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787829343.079683:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829343.079685:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787829343.079688:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829343.079689:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829343.079691:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829343.079693:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787829343.079695:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787829343.079697:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829343.079700:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787829343.079703:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787829343.079707:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d27c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646820608:192.168.203.143@tcp:8:kworker.0 00000100:00000001:1.0:1787829343.079711:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829343.079726:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829343.079728:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787829343.079729:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787829343.079732:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787829343.079734:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787829343.079735:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787829343.079738:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787829343.079739:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787829343.079743:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787829343.079744:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829343.079746:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829343.079747:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787829343.079748:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787829343.079751:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787829343.079752:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787829343.079753:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787829343.079754:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787829343.079755:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787829343.079756:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787829343.079757:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829343.079758:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829343.079759:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787829343.079759:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787829343.079760:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787829343.079761:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787829343.079762:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787829343.079763:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787829343.079764:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787829343.079766:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787829343.079767:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829343.079769:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829343.079773:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787829343.079774:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829343.079776:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787829343.079778:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829343.974697:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829343.974714:0:815:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1787829343.974717:0:815:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1787829343.974720:0:815:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829343.974725:0:815:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1787829343.974727:0:815:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829343.974729:0:815:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829343.974732:0:815:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829343.974790:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829343.974795:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829343.974799:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829344.998677:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829344.998684:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829344.998751:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829344.998755:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829344.998759:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829346.022765:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829346.022770:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829346.022830:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829346.022834:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829346.022837:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829347.046705:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829347.046717:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829347.046781:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829347.046786:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829347.046789:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829348.070724:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829348.070731:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829348.070807:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829348.070812:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829348.070816:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787829348.198713:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829348.198724:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1787829348.198726:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787829348.198729:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787829348.198734:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787829348.198741:0:468:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 4148 00000100:00080000:2.0:1787829348.198752:0:468:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787829348.198755:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829348.198757:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787829348.198762:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787829348.198765:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787829348.198768:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:2.0:1787829348.198770:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:2.0:1787829348.198773:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829348.198774:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:2.0:1787829348.198779:0:468:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787829348.198781:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829348.198784:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787829348.198787:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787829348.198790:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829348.198792:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:2.0:1787829348.198795:0:468:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829348.198807:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787829348.198808:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829348.198811:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787829348.198813:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787829348.198816:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829348.198825:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787829348.198832:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2d680 x1874670646820736/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787829348.198839:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787829348.198843:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829348.198854:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787829348.198857:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829348.198861:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1787829348.198862:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787829348.198864:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787829348.198867:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787829348.198870:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4148 00000100:00080000:2.0:1787829348.198874:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787829348.198876:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829348.198878:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:1.0:1787829348.198880:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:2.0:1787829348.198880:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787829348.198882:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:1.0:1787829348.198884:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000020:00000001:2.0:1787829348.198884:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:1.0:1787829348.198886:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000020:00000001:2.0:1787829348.198886:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:2.0:1787829348.198888:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787829348.198889:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787829348.198889:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:1.0:1787829348.198891:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787829348.198893:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787829348.198895:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829348.198896:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787829348.198897:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1787829348.198899:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829348.198901:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829348.198902:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787829348.198904:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829348.198906:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2d680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646820736:192.168.203.143@tcp:250:kworker.0 00000100:00000001:2.0:1787829348.198906:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:1.0:1787829348.198910:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000008:00100000:2.0:1787829348.198911:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1787829348.198913:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829348.198914:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787829348.198916:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829348.198916:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829348.198919:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1787829348.198920:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787829348.198921:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829348.198922:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829348.198923:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787829348.198924:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787829348.198926:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829348.198932:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829348.198936:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787829348.198937:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417874048 : -126234291677568 : ffff8d30c9e2d680) 00000100:00100000:2.0:1787829348.198937:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2e1c0 x1874670646820864/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787829348.198942:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:2.0:1787829348.198945:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787829348.198947:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829348.198950:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787829348.198953:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829348.198957:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1787829348.198958:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787829348.198959:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787829348.198961:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787829348.198963:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4148 00000100:00080000:2.0:1787829348.198967:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787829348.198969:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829348.198970:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000800:00000100:1.0:1787829348.198972:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000020:00000001:2.0:1787829348.198973:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787829348.198974:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787829348.198976:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000020:00000001:2.0:1787829348.198978:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:2.0:1787829348.198980:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829348.198981:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:1.0:1787829348.198983:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00080000:2.0:1787829348.198984:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787829348.198986:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829348.198987:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 02000000:00000001:2.0:1787829348.198988:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1787829348.198989:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829348.198991:0:2390:0:(events.c:68:request_out_callback()) Process leaving 02000000:00000001:2.0:1787829348.198991:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787829348.198993:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829348.198995:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:1.0:1787829348.198996:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000008:00100000:2.0:1787829348.198997:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1787829348.198999:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829348.199000:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4298822400 00000100:00000001:2.0:1787829348.199001:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829348.199005:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787829348.199006:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829348.199008:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 00000100:00000001:2.0:1787829348.199008:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787829348.199010:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787829348.199011:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829348.199015:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4298822400 00000100:00000001:2.0:1787829348.199016:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829348.199019:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 00000100:00100000:2.0:1787829348.199021:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2cb40 x1874670646820992/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787829348.199022:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829348.199025:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829348.199028:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00000001:2.0:1787829348.199029:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787829348.199031:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829348.199033:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829348/real 1787829348] req@ffff8d30c9e2d680 x1874670646820736/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829373 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787829348.199033:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787829348.199036:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829348.199039:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1787829348.199041:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000400:00000001:1.0:1787829348.199043:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00080000:2.0:1787829348.199043:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000400:00000001:1.0:1787829348.199045:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1787829348.199046:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787829348.199048:0:468:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4148 00000100:00000001:1.0:1787829348.199049:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00080000:2.0:1787829348.199051:0:468:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1787829348.199052:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787829348.199054:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829348.199054:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829348.199055:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:1.0:1787829348.199056:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000020:00000001:2.0:1787829348.199057:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:1.0:1787829348.199059:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:2.0:1787829348.199059:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787829348.199061:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00100000:1.0:1787829348.199063:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2d680 x1874670646820736/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829373 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:2.0:1787829348.199063:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:2.0:1787829348.199065:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829348.199067:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:2.0:1787829348.199070:0:468:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:1.0:1787829348.199071:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829348.199073:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829348.199074:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 02000000:00000001:2.0:1787829348.199074:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787829348.199076:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829348.199078:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:2.0:1787829348.199078:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829348.199080:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829348.199081:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829348.199081:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:1.0:1787829348.199083:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000008:00100000:2.0:1787829348.199083:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00000001:1.0:1787829348.199085:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000008:00000001:2.0:1787829348.199085:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829348.199087:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1787829348.199088:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 02000000:00000001:2.0:1787829348.199091:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787829348.199092:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1787829348.199093:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:2.0:1787829348.199094:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:1.0:1787829348.199095:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 02000000:00000001:2.0:1787829348.199096:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787829348.199097:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829348.199099:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2d680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646820736:192.168.203.143@tcp:250:kworker.0 00000100:00000001:2.0:1787829348.199101:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829348.199104:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787829348.199105:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2d2c0 x1874670646821120/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787829348.199107:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829348.199108:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787829348.199111:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 00000100:00000001:2.0:1787829348.199113:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 02000000:00000001:1.0:1787829348.199114:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00000001:2.0:1787829348.199114:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829348.199116:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000100:00000001:2.0:1787829348.199117:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829348.199118:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787829348.199122:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787829348.199124:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787829348.199127:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787829348.199129:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829348.199131:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829348.199139:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787829348.199141:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829348.199143:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787829348.199145:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1787829348.199146:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787829348.199148:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787829348.199149:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829348.199158:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787829348.199160:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829348.199162:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829348.199165:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646820864:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787829348.199167:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787829348.199169:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787829348.199170:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829348.199171:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787829348.199173:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829348.199178:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787829348.199179:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417876928 : -126234291674688 : ffff8d30c9e2e1c0) 00000100:00000001:1.0:1787829348.199183:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1787829348.199200:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787829348.199205:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787829348.199207:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829348.199209:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829348.199210:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787829348.199213:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829348.199218:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298821142 00000100:00000001:1.0:1787829348.199221:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298822400 00000100:00000001:1.0:1787829348.199223:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298821142 00000100:00000001:1.0:1787829348.199225:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298822400 00000100:00000001:1.0:1787829348.199228:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829348.199230:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829348.199232:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787829348.199237:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829348/real 1787829348] req@ffff8d30c9e2e1c0 x1874670646820864/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829373 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787829348.199246:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787829348.199247:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787829348.199249:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787829348.199250:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787829348.199251:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829348.199252:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829348.199253:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787829348.199256:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2e1c0 x1874670646820864/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829373 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787829348.199261:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829348.199262:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787829348.199264:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829348.199265:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829348.199266:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829348.199267:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787829348.199269:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787829348.199270:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829348.199272:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787829348.199273:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787829348.199275:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646820864:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787829348.199278:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787829348.199279:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787829348.199280:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829348.199281:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787829348.199282:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829348.199284:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829348.199286:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646820992:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787829348.199288:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787829348.199289:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787829348.199290:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829348.199291:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787829348.199293:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829348.199296:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787829348.199297:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417871168 : -126234291680448 : ffff8d30c9e2cb40) 00000100:00000001:1.0:1787829348.199300:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1787829348.199308:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787829348.199311:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787829348.199312:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829348.199314:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829348.199315:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787829348.199317:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829348.199319:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298821142 00000100:00000001:1.0:1787829348.199321:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298822400 00000100:00000001:1.0:1787829348.199322:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298821142 00000100:00000001:1.0:1787829348.199324:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298822400 00000100:00000001:1.0:1787829348.199325:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829348.199326:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829348.199327:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787829348.199330:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829348/real 1787829348] req@ffff8d30c9e2cb40 x1874670646820992/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829373 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787829348.199334:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787829348.199335:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787829348.199336:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787829348.199338:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787829348.199338:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829348.199340:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829348.199341:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787829348.199343:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2cb40 x1874670646820992/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829373 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787829348.199347:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829348.199348:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787829348.199349:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829348.199350:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829348.199351:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829348.199352:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787829348.199353:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787829348.199354:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829348.199356:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787829348.199357:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787829348.199360:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646820992:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787829348.199362:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787829348.199363:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787829348.199364:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829348.199365:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787829348.199366:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829348.199367:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829348.199369:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2d2c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646821120:192.168.203.143@tcp:8:kworker.0 00000100:00000001:1.0:1787829348.199371:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787829348.199372:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787829348.199372:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829348.199373:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787829348.199375:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829348.199378:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787829348.199379:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417873088 : -126234291678528 : ffff8d30c9e2d2c0) 00000100:00000001:1.0:1787829348.199381:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1787829348.199386:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787829348.199388:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787829348.199390:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829348.199390:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829348.199391:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787829348.199393:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829348.199395:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298821155 00000100:00000001:1.0:1787829348.199397:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298822400 00000100:00000001:1.0:1787829348.199398:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298821155 00000100:00000001:1.0:1787829348.199400:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298822400 00000100:00000001:1.0:1787829348.199401:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829348.199402:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829348.199403:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787829348.199405:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829348/real 1787829348] req@ffff8d30c9e2d2c0 x1874670646821120/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829373 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787829348.199414:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787829348.199415:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787829348.199416:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787829348.199417:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787829348.199418:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829348.199419:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829348.199420:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787829348.199422:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2d2c0 x1874670646821120/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829373 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787829348.199426:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829348.199427:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787829348.199429:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829348.199429:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829348.199430:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829348.199431:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787829348.199432:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787829348.199434:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829348.199435:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787829348.199436:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787829348.199438:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2d2c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646821120:192.168.203.143@tcp:8:kworker.0 00000100:00000001:1.0:1787829348.199440:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829348.199442:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829348.199442:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787829348.199443:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787829348.199445:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787829348.199446:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787829348.199447:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787829348.199448:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787829348.199449:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787829348.199452:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787829348.199453:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829348.199454:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829348.199455:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787829348.199456:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787829348.199457:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787829348.199458:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787829348.199459:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787829348.199460:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787829348.199461:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787829348.199462:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787829348.199463:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829348.199464:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829348.199465:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787829348.199465:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787829348.199466:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787829348.199467:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787829348.199468:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787829348.199469:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787829348.199470:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787829348.199472:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787829348.199473:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829348.199474:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829348.199478:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787829348.199479:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829348.199481:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787829348.199482:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829349.094731:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829349.094737:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829349.094802:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829349.094808:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829349.094812:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829350.118744:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829350.118752:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829350.118824:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829350.118829:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829350.118833:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829351.142706:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829351.142713:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829351.142763:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829351.142767:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829351.142770:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829352.166723:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829352.166730:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829352.166797:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829352.166804:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829352.166809:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829353.190723:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829353.190730:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829353.190783:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829353.190789:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829353.190793:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787829353.318794:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829353.318812:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1787829353.318814:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787829353.318818:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787829353.318823:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787829353.318830:0:468:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 4153 00000100:00080000:0.0:1787829353.318841:0:468:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787829353.318844:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829353.318846:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787829353.318851:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787829353.318853:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787829353.318857:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:0.0:1787829353.318860:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:0.0:1787829353.318862:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829353.318863:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:0.0:1787829353.318868:0:468:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787829353.318871:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829353.318873:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787829353.318877:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787829353.318881:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829353.318883:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1787829353.318885:0:468:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829353.318896:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787829353.318898:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829353.318901:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787829353.318904:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787829353.318907:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829353.318918:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787829353.318926:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d270c0 x1874670646821248/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787829353.318936:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787829353.318940:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829353.318955:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787829353.318958:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829353.318962:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1787829353.318964:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787829353.318966:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787829353.318969:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787829353.318971:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4153 00000100:00080000:0.0:1787829353.318975:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787829353.318977:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829353.318979:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787829353.318982:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787829353.318984:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787829353.318986:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000020:00000001:0.0:1787829353.318988:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:1.0:1787829353.318991:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787829353.318995:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787829353.318998:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787829353.318998:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1787829353.319001:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:1.0:1787829353.319002:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787829353.319006:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787829353.319008:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 02000000:00000001:1.0:1787829353.319011:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:0.0:1787829353.319012:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829353.319015:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787829353.319016:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1787829353.319020:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829353.319023:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787829353.319027:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829353.319027:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d270c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646821248:192.168.203.143@tcp:250:kworker.0 00000100:00000001:0.0:1787829353.319032:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:1.0:1787829353.319034:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000008:00100000:0.0:1787829353.319038:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1787829353.319042:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829353.319044:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829353.319047:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787829353.319049:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1787829353.319051:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829353.319053:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829353.319055:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:0.0:1787829353.319058:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1787829353.319061:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829353.319062:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787829353.319066:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829353.319074:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829353.319076:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787829353.319080:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601348800 : -126234108202816 : ffff8d30d4d270c0) 00000100:00100000:0.0:1787829353.319083:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d25a40 x1874670646821376/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787829353.319084:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:0.0:1787829353.319095:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787829353.319098:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829353.319102:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787829353.319105:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829353.319109:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1787829353.319111:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787829353.319113:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787829353.319116:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787829353.319119:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4153 00000800:00000100:1.0:1787829353.319124:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00080000:0.0:1787829353.319126:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787829353.319130:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829353.319132:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:1.0:1787829353.319134:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000020:00000001:0.0:1787829353.319136:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:1.0:1787829353.319138:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000020:00000001:0.0:1787829353.319140:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:1.0:1787829353.319142:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787829353.319144:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:1.0:1787829353.319145:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000020:00000001:0.0:1787829353.319149:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:1.0:1787829353.319151:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829353.319153:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787829353.319155:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4298827520 00000100:00000001:0.0:1787829353.319157:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:0.0:1787829353.319165:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:1.0:1787829353.319166:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1787829353.319170:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829353.319173:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1787829353.319177:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4298827520 02000000:00000001:0.0:1787829353.319178:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787829353.319183:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829353.319185:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1787829353.319186:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1787829353.319191:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00000001:1.0:1787829353.319193:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000008:00000001:0.0:1787829353.319196:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829353.319199:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829353.319200:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829353.319204:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 02000000:00000001:0.0:1787829353.319208:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787829353.319212:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829353.319214:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829353/real 1787829353] req@ffff8d30d4d270c0 x1874670646821248/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829378 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787829353.319217:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787829353.319220:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787829353.319223:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829353.319232:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:1.0:1787829353.319233:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787829353.319236:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787829353.319240:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00100000:0.0:1787829353.319244:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d24f00 x1874670646821504/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787829353.319244:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787829353.319247:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829353.319252:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829353.319254:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787829353.319262:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d270c0 x1874670646821248/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829378 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787829353.319264:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787829353.319278:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829353.319279:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829353.319281:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787829353.319285:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829353.319286:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829353.319291:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00080000:0.0:1787829353.319293:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1787829353.319293:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829353.319297:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787829353.319299:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787829353.319301:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787829353.319303:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00000001:0.0:1787829353.319304:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1787829353.319307:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00080000:0.0:1787829353.319309:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787829353.319310:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000001:0.0:1787829353.319316:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00100000:1.0:1787829353.319316:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d270c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646821248:192.168.203.143@tcp:250:kworker.0 00000100:00000001:1.0:1787829353.319323:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:0.0:1787829353.319324:0:468:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4153 00000100:00000001:1.0:1787829353.319326:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829353.319330:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787829353.319332:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 00000100:00080000:0.0:1787829353.319333:0:468:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 02000000:00000001:1.0:1787829353.319334:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00000001:0.0:1787829353.319338:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000020:00000001:1.0:1787829353.319339:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787829353.319343:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 00000100:00000001:0.0:1787829353.319344:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 02000000:00000001:1.0:1787829353.319347:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 00000020:00000001:0.0:1787829353.319348:0:468:0:(genops.c:792:class_conn2export()) Process entered 02000000:00000001:1.0:1787829353.319349:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000020:00000001:0.0:1787829353.319354:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:1.0:1787829353.319354:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787829353.319357:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1787829353.319358:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:1.0:1787829353.319359:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829353.319365:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787829353.319367:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829353.319368:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787829353.319369:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1787829353.319371:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000020:00000001:0.0:1787829353.319374:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:1.0:1787829353.319375:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787829353.319377:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829353.319380:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 02000000:00000001:1.0:1787829353.319381:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:0.0:1787829353.319384:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 02000000:00000001:1.0:1787829353.319384:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829353.319388:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787829353.319393:0:468:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00100000:1.0:1787829353.319393:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d25a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646821376:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787829353.319400:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829353.319401:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787829353.319404:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787829353.319406:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1787829353.319407:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829353.319409:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1787829353.319411:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829353.319412:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829353.319418:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829353.319421:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787829353.319422:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:1.0:1787829353.319423:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601343040 : -126234108208576 : ffff8d30d4d25a40) 00000100:00000001:1.0:1787829353.319428:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000008:00100000:0.0:1787829353.319431:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1787829353.319436:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829353.319438:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:1.0:1787829353.319441:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 02000000:00000001:0.0:1787829353.319446:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:1.0:1787829353.319446:0:2390:0:(events.c:34:request_out_callback()) Process entered 02000000:00000001:0.0:1787829353.319450:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829353.319451:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829353.319454:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829353.319455:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:1.0:1787829353.319457:0:2390:0:(events.c:68:request_out_callback()) Process leaving 02000000:00000001:0.0:1787829353.319458:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:1.0:1787829353.319460:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829353.319462:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829353.319465:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298826262 00000100:00000001:0.0:1787829353.319469:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829353.319470:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298827520 00000100:00000001:1.0:1787829353.319475:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298826262 00000100:00000001:1.0:1787829353.319477:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298827520 00000100:00000001:1.0:1787829353.319480:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00100000:0.0:1787829353.319481:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d24000 x1874670646821632/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787829353.319482:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829353.319487:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787829353.319493:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829353/real 1787829353] req@ffff8d30d4d25a40 x1874670646821376/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829378 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787829353.319494:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787829353.319498:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829353.319504:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:1.0:1787829353.319505:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787829353.319507:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787829353.319509:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787829353.319511:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787829353.319513:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829353.319514:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829353.319515:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787829353.319518:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d25a40 x1874670646821376/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829378 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787829353.319523:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829353.319525:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787829353.319527:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829353.319529:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829353.319530:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829353.319531:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787829353.319533:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787829353.319534:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829353.319537:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787829353.319539:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787829353.319541:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d25a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646821376:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787829353.319544:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787829353.319544:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787829353.319545:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829353.319546:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787829353.319548:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829353.319549:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829353.319551:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d24f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646821504:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787829353.319552:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787829353.319554:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787829353.319555:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829353.319556:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787829353.319558:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829353.319561:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787829353.319562:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601340160 : -126234108211456 : ffff8d30d4d24f00) 00000100:00000001:1.0:1787829353.319564:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1787829353.319570:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787829353.319573:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787829353.319574:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829353.319575:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829353.319576:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787829353.319578:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829353.319580:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298826262 00000100:00000001:1.0:1787829353.319582:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298827520 00000100:00000001:1.0:1787829353.319583:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298826262 00000100:00000001:1.0:1787829353.319585:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298827520 00000100:00000001:1.0:1787829353.319586:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829353.319588:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829353.319589:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787829353.319592:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829353/real 1787829353] req@ffff8d30d4d24f00 x1874670646821504/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829378 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787829353.319597:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787829353.319598:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787829353.319628:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787829353.319630:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787829353.319631:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829353.319632:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829353.319633:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787829353.319636:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d24f00 x1874670646821504/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829378 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787829353.319657:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829353.319658:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787829353.319660:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829353.319661:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829353.319662:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829353.319663:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787829353.319664:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787829353.319666:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829353.319668:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787829353.319670:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787829353.319672:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d24f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646821504:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787829353.319675:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829353.319676:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829353.319677:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787829353.319678:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787829353.319680:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787829353.319681:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787829353.319682:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787829353.319684:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787829353.319685:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787829353.319687:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787829353.319688:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829353.319690:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829353.319690:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787829353.319691:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787829353.319693:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787829353.319694:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787829353.319694:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787829353.319695:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787829353.319696:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787829353.319698:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787829353.319698:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829353.319700:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829353.319702:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787829353.319703:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829353.319704:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787829353.319705:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1787829353.319706:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787829353.319706:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787829353.319707:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829353.319709:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787829353.319710:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829353.319712:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829353.319714:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d24000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646821632:192.168.203.143@tcp:8:kworker.0 00000100:00000001:1.0:1787829353.319716:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787829353.319717:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787829353.319718:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829353.319720:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787829353.319722:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829353.319726:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787829353.319727:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601336320 : -126234108215296 : ffff8d30d4d24000) 00000100:00000001:1.0:1787829353.319730:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1787829353.319738:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787829353.319741:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787829353.319743:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829353.319744:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829353.319745:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787829353.319746:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829353.319749:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298826276 00000100:00000001:1.0:1787829353.319751:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298827520 00000100:00000001:1.0:1787829353.319753:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298826276 00000100:00000001:1.0:1787829353.319754:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298827520 00000100:00000001:1.0:1787829353.319756:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829353.319757:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829353.319758:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787829353.319760:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829353/real 1787829353] req@ffff8d30d4d24000 x1874670646821632/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829378 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787829353.319765:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787829353.319766:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787829353.319767:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787829353.319768:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787829353.319769:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829353.319770:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829353.319771:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787829353.319773:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d24000 x1874670646821632/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829378 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787829353.319778:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829353.319779:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787829353.319781:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829353.319782:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829353.319783:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829353.319784:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787829353.319785:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787829353.319786:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829353.319788:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787829353.319789:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787829353.319791:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d24000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646821632:192.168.203.143@tcp:8:kworker.0 00000100:00000001:1.0:1787829353.319793:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829353.319795:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829353.319796:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787829353.319797:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787829353.319798:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787829353.319799:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787829353.319800:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787829353.319801:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787829353.319802:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787829353.319803:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787829353.319804:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829353.319805:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829353.319806:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787829353.319807:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829353.319809:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787829353.319810:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829354.214723:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829354.214730:0:815:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1787829354.214732:0:815:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1787829354.214734:0:815:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829354.214739:0:815:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1787829354.214740:0:815:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829354.214742:0:815:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829354.214744:0:815:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829354.214835:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829354.214840:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829354.214843:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829355.238724:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829355.238729:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829355.238791:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829355.238797:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829355.238801:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829356.262730:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829356.262742:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829356.262839:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829356.262846:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829356.262852:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829357.286705:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829357.286713:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829357.286766:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829357.286771:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829357.286776:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829358.310724:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829358.310735:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829358.310799:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829358.310808:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829358.310811:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787829358.438698:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829358.438710:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1787829358.438713:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787829358.438716:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787829358.438720:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787829358.438727:0:468:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 4158 00000100:00080000:2.0:1787829358.438739:0:468:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787829358.438742:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829358.438743:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787829358.438748:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787829358.438751:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787829358.438754:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:2.0:1787829358.438756:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:2.0:1787829358.438758:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829358.438759:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:2.0:1787829358.438763:0:468:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787829358.438764:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829358.438767:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787829358.438770:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787829358.438773:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829358.438775:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:2.0:1787829358.438777:0:468:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829358.438790:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787829358.438792:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829358.438795:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787829358.438798:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787829358.438810:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829358.438823:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787829358.438834:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2c780 x1874670646821760/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787829358.438844:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787829358.438848:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829358.438860:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787829358.438863:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829358.438865:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1787829358.438866:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787829358.438867:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787829358.438869:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787829358.438871:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4158 00000100:00080000:2.0:1787829358.438874:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787829358.438876:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829358.438879:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787829358.438881:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787829358.438882:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787829358.438883:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:1.0:1787829358.438886:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:2.0:1787829358.438886:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:2.0:1787829358.438890:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829358.438891:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:1.0:1787829358.438892:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00080000:2.0:1787829358.438894:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:1.0:1787829358.438896:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1787829358.438897:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829358.438901:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1787829358.438902:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787829358.438907:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829358.438907:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787829358.438912:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829358.438913:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 02000000:00000001:1.0:1787829358.438915:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000008:00100000:2.0:1787829358.438918:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 02000000:00000001:1.0:1787829358.438920:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000008:00000001:2.0:1787829358.438922:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829358.438927:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829358.438928:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829358.438931:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787829358.438933:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829358.438936:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00100000:1.0:1787829358.438938:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646821760:192.168.203.143@tcp:250:kworker.0 02000000:00000001:2.0:1787829358.438939:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787829358.438941:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829358.438948:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:2.0:1787829358.438948:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829358.438955:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000100:00100000:2.0:1787829358.438956:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2c3c0 x1874670646821888/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:1.0:1787829358.438958:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829358.438964:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:2.0:1787829358.438968:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787829358.438970:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829358.438972:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829358.438972:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787829358.438978:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829358.438980:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1787829358.438981:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787829358.438982:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787829358.438984:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787829358.438987:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4158 00000100:00080000:2.0:1787829358.438990:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787829358.438991:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829358.438992:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787829358.438993:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787829358.438994:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787829358.438995:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000020:00000001:2.0:1787829358.438997:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:1.0:1787829358.438999:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787829358.438999:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829358.439003:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:1.0:1787829358.439004:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417870208 : -126234291681408 : ffff8d30c9e2c780) 00000100:00080000:2.0:1787829358.439006:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787829358.439010:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829358.439012:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1787829358.439013:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 02000000:00000001:2.0:1787829358.439014:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787829358.439017:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829358.439018:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1787829358.439020:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1787829358.439021:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829358.439023:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829358.439026:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787829358.439027:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829358.439028:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787829358.439029:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787829358.439031:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829358.439034:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787829358.439037:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2de00 x1874670646822016/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787829358.439042:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787829358.439044:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829358.439045:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787829358.439047:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829358.439049:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1787829358.439051:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787829358.439052:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787829358.439054:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787829358.439056:0:468:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4158 00000100:00080000:2.0:1787829358.439059:0:468:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787829358.439060:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829358.439061:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787829358.439062:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787829358.439063:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787829358.439064:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:1.0:1787829358.439066:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1787829358.439070:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:1.0:1787829358.439071:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4298832640 00000100:00000001:2.0:1787829358.439075:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829358.439077:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:2.0:1787829358.439079:0:468:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787829358.439081:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829358.439083:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1787829358.439084:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 02000000:00000001:2.0:1787829358.439086:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787829358.439089:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829358.439091:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1787829358.439093:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00000001:1.0:1787829358.439094:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4298832640 00000008:00000001:2.0:1787829358.439095:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829358.439100:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829358.439102:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 02000000:00000001:2.0:1787829358.439104:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:1.0:1787829358.439107:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787829358.439107:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829358.439109:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:1.0:1787829358.439113:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829358.439113:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787829358.439115:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829358.439118:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829358.439121:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829358.439122:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787829358.439130:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2fc00 x1874670646822144/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787829358.439134:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1787829358.439136:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787829358.439139:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:2.0:1787829358.439139:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829358.439146:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787829358.439146:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829358.439151:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1787829358.439155:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787829358.439157:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787829358.439158:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829358.439162:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787829358.439165:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829358.439168:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829358.439174:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646821888:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787829358.439178:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787829358.439181:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787829358.439182:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829358.439184:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787829358.439187:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829358.439193:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787829358.439194:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417869248 : -126234291682368 : ffff8d30c9e2c3c0) 00000100:00000001:1.0:1787829358.439199:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:1.0:1787829358.439210:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829358.439214:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298831382 00000100:00000001:1.0:1787829358.439216:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298832640 00000100:00000001:1.0:1787829358.439218:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298831382 00000100:00000001:1.0:1787829358.439220:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298832640 00000100:00000001:1.0:1787829358.439221:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829358.439223:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829358.439224:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787829358.439225:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787829358.439226:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829358.439228:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787829358.439229:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829358.439231:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829358.439234:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646822016:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787829358.439236:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787829358.439237:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787829358.439238:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829358.439239:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787829358.439241:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829358.439245:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787829358.439246:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417875968 : -126234291675648 : ffff8d30c9e2de00) 00000100:00000001:1.0:1787829358.439248:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:1.0:1787829358.439255:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829358.439258:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298831382 00000100:00000001:1.0:1787829358.439259:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298832640 00000100:00000001:1.0:1787829358.439262:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298831382 00000100:00000001:1.0:1787829358.439264:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298832640 00000100:00000001:1.0:1787829358.439265:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829358.439266:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829358.439268:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787829358.439268:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787829358.439269:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829358.439270:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787829358.439271:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829358.439273:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829358.439275:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646822144:192.168.203.143@tcp:8:kworker.0 00000100:00000001:1.0:1787829358.439277:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787829358.439278:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787829358.439279:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829358.439280:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787829358.439282:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829358.439285:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787829358.439286:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417883648 : -126234291667968 : ffff8d30c9e2fc00) 00000100:00000001:1.0:1787829358.439288:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:1.0:1787829358.439292:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829358.439294:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298831395 00000100:00000001:1.0:1787829358.439296:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298832640 00000100:00000001:1.0:1787829358.439298:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298831395 00000100:00000001:1.0:1787829358.439299:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298832640 00000100:00000001:1.0:1787829358.439301:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829358.439302:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829358.439303:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829358.439304:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829358.439307:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787829358.439307:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:1.0:1787829358.439309:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787829358.439310:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1787829358.439311:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829358.439312:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829359.334782:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829359.334792:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829359.334871:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829359.334876:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829359.334880:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829360.358695:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829360.358701:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829360.358787:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829360.358794:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829360.358797:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829361.382751:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829361.382762:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829361.382848:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829361.382854:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829361.382857:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:3.0:1787829361.510846:0:2379:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.143:988 00000400:00000100:3.0:1787829361.510883:0:2379:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.143@tcp at host 192.168.203.143 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:3.0:1787829361.510901:0:2379:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.143@tcp 00000100:00000001:3.0:1787829361.510915:0:2379:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787829361.510933:0:2379:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829361.510936:0:2379:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829361.510938:0:2379:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:3.0:1787829361.510948:0:2379:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.143@tcp 00000100:00000001:3.0:1787829361.510953:0:2379:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787829361.510957:0:2379:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829361.510961:0:2379:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829361.510963:0:2379:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:3.0:1787829361.510969:0:2379:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.143@tcp 00000100:00000001:3.0:1787829361.510973:0:2379:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787829361.510981:0:2379:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829361.510983:0:2379:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829361.510984:0:2379:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787829361.510988:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000800:00000100:3.0:1787829361.510990:0:2379:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.143@tcp 00000100:00000001:1.0:1787829361.510999:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787829361.511002:0:2379:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787829361.511006:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00000001:3.0:1787829361.511010:0:2379:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829361.511014:0:2379:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829361.511017:0:2379:0:(events.c:68:request_out_callback()) Process leaving 00000100:00100000:1.0:1787829361.511030:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829358/real 1787829361] req@ffff8d30c9e2c3c0 x1874670646821888/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829383 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787829361.511054:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787829361.511057:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787829361.511067:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787829361.511072:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787829361.511074:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829361.511078:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829361.511080:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787829361.511095:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2c3c0 x1874670646821888/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829383 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787829361.511105:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829361.511110:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787829361.511114:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829361.511116:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829361.511119:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829361.511122:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787829361.511126:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787829361.511130:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829361.511136:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787829361.511140:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787829361.511145:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646821888:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787829361.511151:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787829361.511156:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829358/real 1787829361] req@ffff8d30c9e2de00 x1874670646822016/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829383 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787829361.511168:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787829361.511169:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787829361.511173:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787829361.511175:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787829361.511177:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829361.511179:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829361.511181:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787829361.511185:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2de00 x1874670646822016/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829383 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787829361.511193:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829361.511195:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787829361.511197:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829361.511199:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829361.511201:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829361.511203:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787829361.511205:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787829361.511208:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829361.511212:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787829361.511215:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787829361.511219:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646822016:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787829361.511224:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787829361.511229:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829358/real 1787829361] req@ffff8d30c9e2fc00 x1874670646822144/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829383 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787829361.511244:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787829361.511245:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787829361.511249:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787829361.511251:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787829361.511253:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829361.511255:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829361.511257:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787829361.511261:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2fc00 x1874670646822144/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829383 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787829361.511270:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829361.511272:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787829361.511275:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829361.511277:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829361.511278:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829361.511280:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787829361.511283:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787829361.511285:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829361.511289:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787829361.511292:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787829361.511295:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646822144:192.168.203.143@tcp:8:kworker.0 00000100:00000001:1.0:1787829361.511301:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787829361.511306:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829358/real 1787829361] req@ffff8d30c9e2c780 x1874670646821760/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829383 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787829361.511315:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787829361.511317:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787829361.511321:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787829361.511323:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787829361.511324:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829361.511326:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829361.511327:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787829361.511332:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2c780 x1874670646821760/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829383 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787829361.511340:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829361.511342:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787829361.511345:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829361.511346:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829361.511348:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829361.511350:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787829361.511352:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787829361.511355:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829361.511358:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787829361.511361:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787829361.511364:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646821760:192.168.203.143@tcp:250:kworker.0 00000100:00000001:1.0:1787829361.511369:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829361.511373:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829361.511375:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787829361.511379:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787829361.511387:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787829361.511391:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787829361.511394:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787829361.511399:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787829361.511402:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787829361.511407:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787829361.511408:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829361.511411:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829361.511412:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787829361.511414:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787829361.511416:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787829361.511418:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787829361.511419:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787829361.511422:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787829361.511423:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787829361.511430:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787829361.511431:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829361.511433:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829361.511434:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787829361.511435:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787829361.511438:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787829361.511439:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787829361.511440:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787829361.511442:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787829361.511444:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787829361.511447:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787829361.511448:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829361.511451:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829361.511452:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787829361.511454:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787829361.511456:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787829361.511457:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787829361.511459:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787829361.511460:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787829361.511462:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787829361.511463:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787829361.511465:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829361.511468:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829361.511479:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787829361.511482:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829361.511485:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787829361.511487:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829362.406722:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829362.406729:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829362.406924:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829362.406930:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829362.406936:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829363.430733:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829363.430742:0:815:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1787829363.430745:0:815:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1787829363.430748:0:815:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829363.430754:0:815:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1787829363.430757:0:815:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829363.430760:0:815:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829363.430763:0:815:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829363.430848:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829363.430853:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829363.430857:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787829363.558725:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829363.558734:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1787829363.558737:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787829363.558742:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787829363.558747:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787829363.558757:0:468:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 4164 00000100:00080000:2.0:1787829363.558770:0:468:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787829363.558779:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829363.558782:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787829363.558789:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787829363.558792:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787829363.558795:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:2.0:1787829363.558797:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:2.0:1787829363.558799:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829363.558800:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:2.0:1787829363.558804:0:468:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787829363.558808:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829363.558811:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787829363.558815:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787829363.558817:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829363.558819:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:2.0:1787829363.558821:0:468:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829363.558834:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787829363.558835:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829363.558839:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787829363.558842:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787829363.558846:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829363.558859:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787829363.558872:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2f0c0 x1874670646822272/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787829363.558885:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787829363.558891:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829363.558914:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787829363.558918:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829363.558922:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1787829363.558924:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787829363.558926:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787829363.558931:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787829363.558935:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4164 00000100:00080000:2.0:1787829363.558940:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787829363.558942:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787829363.558944:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829363.558948:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:3.0:1787829363.558951:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000020:00000001:2.0:1787829363.558953:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:3.0:1787829363.558957:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000020:00000001:2.0:1787829363.558958:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787829363.558963:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000020:00000001:2.0:1787829363.558967:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:2.0:1787829363.558969:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829363.558971:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787829363.558975:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829363.558982:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:2.0:1787829363.558983:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 02000000:00000001:3.0:1787829363.558991:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00080000:2.0:1787829363.558995:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 02000000:00000001:3.0:1787829363.558997:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829363.559007:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829363.559012:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646822272:192.168.203.143@tcp:250:kworker.0 02000000:00000001:2.0:1787829363.559014:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:3.0:1787829363.559024:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787829363.559027:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829363.559030:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1787829363.559035:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829363.559037:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829363.559042:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 02000000:00000001:3.0:1787829363.559044:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000008:00100000:2.0:1787829363.559055:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 02000000:00000001:3.0:1787829363.559055:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:2.0:1787829363.559066:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829363.559071:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829363.559081:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:3.0:1787829363.559083:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 02000000:00000001:2.0:1787829363.559084:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829363.559089:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417880768 : -126234291670848 : ffff8d30c9e2f0c0) 00000100:00000001:2.0:1787829363.559090:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787829363.559094:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787829363.559099:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829363.559099:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:2.0:1787829363.559115:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787829363.559124:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2ed00 x1874670646822400/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787829363.559134:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787829363.559137:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829363.559143:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787829363.559147:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829363.559151:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1787829363.559153:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787829363.559155:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:3.0:1787829363.559161:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:2.0:1787829363.559172:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787829363.559178:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4164 00000100:00000001:3.0:1787829363.559183:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00080000:2.0:1787829363.559185:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787829363.559190:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829363.559191:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787829363.559193:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:3.0:1787829363.559197:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1787829363.559199:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787829363.559206:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:3.0:1787829363.559206:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000020:00000001:2.0:1787829363.559214:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:3.0:1787829363.559218:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1787829363.559219:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:2.0:1787829363.559228:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829363.559230:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4298837760 00000100:00000001:2.0:1787829363.559231:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:2.0:1787829363.559243:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:3.0:1787829363.559248:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 00000100:00000001:2.0:1787829363.559249:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829363.559258:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787829363.559263:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787829363.559266:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829363.559268:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1787829363.559271:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00000001:3.0:1787829363.559272:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4298837760 00000008:00000001:2.0:1787829363.559278:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829363.559284:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829363.559284:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 02000000:00000001:2.0:1787829363.559298:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:3.0:1787829363.559299:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787829363.559303:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829363.559306:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829363.559310:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:3.0:1787829363.559313:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 02000000:00000001:2.0:1787829363.559315:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787829363.559319:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829363.559327:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829363.559329:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829363/real 1787829363] req@ffff8d30c9e2f0c0 x1874670646822272/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829388 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829363.559340:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2e940 x1874670646822528/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000001:3.0:1787829363.559368:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000001:2.0:1787829363.559370:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000400:00000001:3.0:1787829363.559374:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1787829363.559378:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829363.559384:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1787829363.559386:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829363.559391:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00080000:2.0:1787829363.559392:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1787829363.559399:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:2.0:1787829363.559402:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1787829363.559408:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787829363.559409:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787829363.559417:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787829363.559417:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829363.559429:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00100000:3.0:1787829363.559439:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2f0c0 x1874670646822272/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829388 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829363.559445:0:468:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4164 00000100:00080000:2.0:1787829363.559457:0:468:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787829363.559462:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829363.559463:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787829363.559466:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787829363.559468:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787829363.559470:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000020:00000001:2.0:1787829363.559472:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:2.0:1787829363.559474:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829363.559475:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:2.0:1787829363.559481:0:468:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787829363.559484:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829363.559486:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787829363.559490:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787829363.559492:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829363.559495:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1787829363.559497:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000008:00000001:2.0:1787829363.559500:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829363.559502:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829363.559506:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787829363.559508:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829363.559510:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787829363.559511:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787829363.559513:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829363.559520:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787829363.559525:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2c000 x1874670646822656/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787829363.559533:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787829363.559535:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829363.559537:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829363.559673:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829363.559677:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829363.559681:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829363.559683:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829363.559685:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829363.559688:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829363.559691:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829363.559694:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829363.559702:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829363.559704:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829363.559709:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646822272:192.168.203.143@tcp:250:kworker.0 00000100:00000001:3.0:1787829363.559714:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829363.559717:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829363.559718:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829363.559721:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829363.559725:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829363.559738:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829363.559740:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829363.559745:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829363.559746:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829363.559750:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829363.559752:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829363.559755:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829363.559762:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787829363.559764:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829363.559767:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1787829363.559769:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787829363.559771:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787829363.559773:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787829363.559775:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829363.559778:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787829363.559781:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829363.559784:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829363.559788:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646822400:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829363.559791:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1787829363.559794:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1787829363.559796:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829363.559799:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1787829363.559802:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829363.559811:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787829363.559813:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417879808 : -126234291671808 : ffff8d30c9e2ed00) 00000100:00000001:3.0:1787829363.559819:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1787829363.559840:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:3.0:1787829363.559846:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787829363.559849:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829363.559851:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829363.559852:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1787829363.559856:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829363.559861:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298836503 00000100:00000001:3.0:1787829363.559864:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298837760 00000100:00000001:3.0:1787829363.559867:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298836503 00000100:00000001:3.0:1787829363.559870:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298837760 00000100:00000001:3.0:1787829363.559871:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829363.559874:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829363.559876:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1787829363.559881:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829363/real 1787829363] req@ffff8d30c9e2ed00 x1874670646822400/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829388 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1787829363.559892:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787829363.559894:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829363.559897:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829363.559898:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829363.559900:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829363.559902:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829363.559903:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829363.559907:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2ed00 x1874670646822400/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829388 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829363.559913:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829363.559916:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829363.559918:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829363.559920:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829363.559921:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829363.559923:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829363.559925:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829363.559927:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829363.559945:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829363.559948:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829363.559952:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646822400:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829363.559956:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787829363.559958:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787829363.559960:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829363.559962:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787829363.559964:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829363.559966:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829363.559969:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646822528:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829363.559972:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1787829363.559974:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1787829363.559976:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829363.559978:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1787829363.559981:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829363.559986:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787829363.559988:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417878848 : -126234291672768 : ffff8d30c9e2e940) 00000100:00000001:3.0:1787829363.559991:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1787829363.560005:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:3.0:1787829363.560010:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787829363.560012:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829363.560013:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829363.560015:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1787829363.560017:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829363.560021:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298836503 00000100:00000001:3.0:1787829363.560024:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298837760 00000100:00000001:3.0:1787829363.560026:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298836503 00000100:00000001:3.0:1787829363.560028:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298837760 00000100:00000001:3.0:1787829363.560030:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829363.560033:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829363.560034:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1787829363.560039:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829363/real 1787829363] req@ffff8d30c9e2e940 x1874670646822528/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829388 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1787829363.560048:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787829363.560049:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829363.560053:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829363.560054:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829363.560056:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829363.560057:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829363.560059:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829363.560064:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2e940 x1874670646822528/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829388 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829363.560071:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829363.560073:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829363.560076:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829363.560078:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829363.560079:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829363.560081:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829363.560082:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829363.560086:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829363.560090:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829363.560094:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829363.560098:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646822528:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829363.560103:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787829363.560105:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787829363.560107:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829363.560110:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787829363.560111:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829363.560113:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829363.560116:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646822656:192.168.203.143@tcp:8:kworker.0 00000100:00000001:3.0:1787829363.560120:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1787829363.560123:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1787829363.560124:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829363.560126:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1787829363.560130:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829363.560136:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787829363.560138:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417868288 : -126234291683328 : ffff8d30c9e2c000) 00000100:00000001:3.0:1787829363.560141:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1787829363.560155:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:3.0:1787829363.560162:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787829363.560165:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829363.560167:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829363.560169:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1787829363.560172:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829363.560179:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298836516 00000100:00000001:3.0:1787829363.560181:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298837760 00000100:00000001:3.0:1787829363.560184:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298836516 00000100:00000001:3.0:1787829363.560187:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298837760 00000100:00000001:3.0:1787829363.560189:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829363.560191:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829363.560193:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1787829363.560197:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829363/real 1787829363] req@ffff8d30c9e2c000 x1874670646822656/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829388 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1787829363.560205:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787829363.560207:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829363.560209:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829363.560211:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829363.560212:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829363.560214:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829363.560216:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829363.560219:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2c000 x1874670646822656/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829388 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829363.560226:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829363.560228:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829363.560230:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829363.560231:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829363.560232:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829363.560234:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829363.560236:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829363.560238:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829363.560241:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829363.560244:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829363.560247:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646822656:192.168.203.143@tcp:8:kworker.0 00000100:00000001:3.0:1787829363.560251:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829363.560253:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829363.560255:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829363.560256:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829363.560259:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829363.560261:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829363.560262:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829363.560264:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829363.560266:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829363.560270:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829363.560271:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829363.560273:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829363.560274:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829363.560275:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829363.560277:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829363.560278:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829363.560279:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829363.560281:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829363.560282:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829363.560283:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829363.560285:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829363.560287:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829363.560288:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829363.560290:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829363.560291:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829363.560293:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829363.560294:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829363.560295:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829363.560297:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829363.560299:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829363.560300:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829363.560302:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829363.560306:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787829363.560307:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829363.560309:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1787829363.560310:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829364.454706:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829364.454713:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829364.454760:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829364.454766:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829364.454770:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829365.478733:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829365.478741:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829365.478813:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829365.478817:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829365.478820:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829366.502735:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829366.502742:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829366.502824:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829366.502831:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829366.502837:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829367.526729:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829367.526736:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829367.526825:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829367.526831:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829367.526834:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829368.550754:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829368.550761:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829368.550847:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829368.550851:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829368.550854:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787829368.678772:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829368.678789:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1787829368.678792:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787829368.678797:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787829368.678803:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787829368.678812:0:468:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 4169 00000100:00080000:0.0:1787829368.678826:0:468:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787829368.678832:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829368.678833:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787829368.678838:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787829368.678841:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787829368.678844:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:0.0:1787829368.678845:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:0.0:1787829368.678847:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829368.678848:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:0.0:1787829368.678852:0:468:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787829368.678854:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829368.678857:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787829368.678860:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787829368.678863:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829368.678865:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1787829368.678867:0:468:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829368.678879:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787829368.678881:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829368.678884:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787829368.678887:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787829368.678891:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829368.678905:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787829368.678915:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d24000 x1874670646822784/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787829368.678925:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787829368.678929:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829368.678946:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787829368.678948:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829368.678951:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1787829368.678952:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787829368.678953:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787829368.678955:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787829368.678957:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4169 00000100:00080000:0.0:1787829368.678960:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787829368.678962:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829368.678963:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787829368.678965:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787829368.678966:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787829368.678968:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000020:00000001:0.0:1787829368.678970:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:0.0:1787829368.678971:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829368.678972:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:0.0:1787829368.678974:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787829368.678976:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829368.678978:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787829368.678983:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787829368.678985:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829368.678987:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1787829368.678991:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1787829368.678994:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829368.678995:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829368.678999:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787829368.679000:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829368.679000:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787829368.679002:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787829368.679006:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787829368.679007:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829368.679010:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787829368.679011:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829368.679015:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00100000:0.0:1787829368.679017:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d24f00 x1874670646822912/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787829368.679019:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787829368.679023:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829368.679029:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787829368.679032:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829368.679032:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:0.0:1787829368.679037:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829368.679037:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00080000:0.0:1787829368.679043:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 02000000:00000001:3.0:1787829368.679044:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787829368.679050:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1787829368.679054:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00100000:3.0:1787829368.679054:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d24000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646822784:192.168.203.143@tcp:250:kworker.0 00000100:00080000:0.0:1787829368.679056:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787829368.679063:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:0.0:1787829368.679064:0:468:0:(import.c:485:import_select_connection()) Process entered 02000000:00000001:3.0:1787829368.679070:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000100:00080000:0.0:1787829368.679071:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4169 02000000:00000001:3.0:1787829368.679075:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787829368.679077:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 02000000:00000001:3.0:1787829368.679080:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:0.0:1787829368.679083:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829368.679086:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 02000000:00000001:3.0:1787829368.679088:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787829368.679089:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787829368.679092:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787829368.679096:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000020:00000001:0.0:1787829368.679099:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:0.0:1787829368.679101:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829368.679103:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:0.0:1787829368.679107:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787829368.679110:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829368.679112:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 02000000:00000001:0.0:1787829368.679113:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:3.0:1787829368.679117:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601336320 : -126234108215296 : ffff8d30d4d24000) 02000000:00000001:0.0:1787829368.679118:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787829368.679124:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829368.679127:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:3.0:1787829368.679127:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000008:00100000:0.0:1787829368.679130:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1787829368.679133:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829368.679134:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829368.679138:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787829368.679139:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829368.679140:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787829368.679141:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787829368.679142:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829368.679147:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787829368.679151:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d25a40 x1874670646823040/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787829368.679157:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787829368.679159:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829368.679162:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787829368.679164:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829368.679166:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1787829368.679168:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787829368.679169:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787829368.679171:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787829368.679172:0:468:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4169 00000100:00080000:0.0:1787829368.679177:0:468:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000800:00000100:3.0:1787829368.679178:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:0.0:1787829368.679179:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829368.679184:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787829368.679186:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787829368.679187:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787829368.679188:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000020:00000001:0.0:1787829368.679190:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:0.0:1787829368.679192:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829368.679193:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:0.0:1787829368.679196:0:468:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787829368.679197:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829368.679199:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787829368.679200:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787829368.679203:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829368.679203:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1787829368.679205:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1787829368.679207:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1787829368.679209:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829368.679210:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787829368.679211:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829368.679217:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:3.0:1787829368.679217:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829368.679218:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829368.679223:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:3.0:1787829368.679225:0:2390:0:(events.c:68:request_out_callback()) Process leaving 02000000:00000001:0.0:1787829368.679226:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787829368.679228:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829368.679232:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829368.679236:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787829368.679243:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d270c0 x1874670646823168/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787829368.679245:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4298842880 00000100:00000001:0.0:1787829368.679255:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787829368.679257:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829368.679258:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829368.679263:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 00000100:00000001:3.0:1787829368.679272:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4298842880 00000100:00000001:3.0:1787829368.679279:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 00000100:00000001:3.0:1787829368.679285:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829368.679289:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829368.679293:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1787829368.679300:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829368/real 1787829368] req@ffff8d30d4d24000 x1874670646822784/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829393 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1787829368.679312:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787829368.679314:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829368.679318:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829368.679320:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829368.679322:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829368.679324:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829368.679326:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829368.679330:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d24000 x1874670646822784/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829393 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829368.679336:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829368.679339:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829368.679341:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829368.679343:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829368.679343:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829368.679345:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829368.679347:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829368.679349:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829368.679352:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829368.679354:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829368.679357:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d24000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646822784:192.168.203.143@tcp:250:kworker.0 00000100:00000001:3.0:1787829368.679360:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829368.679362:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829368.679364:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829368.679367:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829368.679370:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829368.679372:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829368.679374:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829368.679377:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829368.679379:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829368.679382:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829368.679383:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829368.679385:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829368.679392:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787829368.679394:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829368.679396:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1787829368.679397:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787829368.679398:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787829368.679399:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787829368.679400:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829368.679402:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787829368.679403:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829368.679405:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829368.679407:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d24f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646822912:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829368.679409:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1787829368.679411:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1787829368.679412:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829368.679413:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1787829368.679416:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829368.679421:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787829368.679422:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601340160 : -126234108211456 : ffff8d30d4d24f00) 00000100:00000001:3.0:1787829368.679426:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1787829368.679437:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:3.0:1787829368.679441:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787829368.679443:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829368.679444:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829368.679445:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1787829368.679447:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829368.679451:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298841622 00000100:00000001:3.0:1787829368.679453:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298842880 00000100:00000001:3.0:1787829368.679455:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298841622 00000100:00000001:3.0:1787829368.679456:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298842880 00000100:00000001:3.0:1787829368.679458:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829368.679459:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829368.679461:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1787829368.679464:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829368/real 1787829368] req@ffff8d30d4d24f00 x1874670646822912/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829393 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1787829368.679470:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787829368.679471:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829368.679472:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829368.679473:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829368.679474:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829368.679476:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829368.679477:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829368.679479:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d24f00 x1874670646822912/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829393 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829368.679483:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829368.679485:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829368.679486:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829368.679487:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829368.679488:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829368.679489:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829368.679490:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829368.679492:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829368.679494:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829368.679495:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829368.679497:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d24f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646822912:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829368.679500:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787829368.679501:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787829368.679502:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829368.679503:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787829368.679504:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829368.679505:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829368.679507:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d25a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646823040:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829368.679511:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1787829368.679512:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1787829368.679513:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829368.679515:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1787829368.679517:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829368.679519:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787829368.679520:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601343040 : -126234108208576 : ffff8d30d4d25a40) 00000100:00000001:3.0:1787829368.679523:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1787829368.679529:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:3.0:1787829368.679532:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787829368.679549:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829368.679550:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829368.679552:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1787829368.679554:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829368.679557:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298841622 00000100:00000001:3.0:1787829368.679559:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298842880 00000100:00000001:3.0:1787829368.679561:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298841622 00000100:00000001:3.0:1787829368.679563:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298842880 00000100:00000001:3.0:1787829368.679564:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829368.679565:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829368.679566:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1787829368.679570:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829368/real 1787829368] req@ffff8d30d4d25a40 x1874670646823040/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829393 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1787829368.679575:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787829368.679575:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829368.679577:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829368.679578:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829368.679579:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829368.679580:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829368.679581:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829368.679584:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d25a40 x1874670646823040/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829393 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829368.679593:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829368.679595:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829368.679597:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829368.679598:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829368.679598:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829368.679624:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829368.679625:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829368.679627:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829368.679629:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829368.679631:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829368.679633:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d25a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646823040:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829368.679636:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787829368.679637:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787829368.679638:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829368.679639:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787829368.679641:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829368.679642:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829368.679644:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d270c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646823168:192.168.203.143@tcp:8:kworker.0 00000100:00000001:3.0:1787829368.679646:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1787829368.679648:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1787829368.679649:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829368.679650:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1787829368.679652:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829368.679656:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787829368.679657:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601348800 : -126234108202816 : ffff8d30d4d270c0) 00000100:00000001:3.0:1787829368.679659:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1787829368.679667:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:3.0:1787829368.679671:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787829368.679672:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829368.679673:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829368.679675:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1787829368.679677:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829368.679679:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298841636 00000100:00000001:3.0:1787829368.679681:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298842880 00000100:00000001:3.0:1787829368.679682:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298841636 00000100:00000001:3.0:1787829368.679684:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298842880 00000100:00000001:3.0:1787829368.679685:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829368.679686:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829368.679688:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1787829368.679690:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829368/real 1787829368] req@ffff8d30d4d270c0 x1874670646823168/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829393 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1787829368.679695:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787829368.679695:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829368.679697:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829368.679698:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829368.679699:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829368.679700:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829368.679701:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829368.679704:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d270c0 x1874670646823168/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829393 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829368.679708:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829368.679709:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829368.679710:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829368.679711:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829368.679712:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829368.679713:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829368.679714:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829368.679716:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829368.679718:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829368.679719:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829368.679722:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d270c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646823168:192.168.203.143@tcp:8:kworker.0 00000100:00000001:3.0:1787829368.679725:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829368.679726:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829368.679727:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829368.679728:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829368.679729:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829368.679730:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829368.679731:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829368.679733:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829368.679733:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829368.679735:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829368.679736:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829368.679737:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829368.679738:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829368.679739:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829368.679740:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829368.679741:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829368.679742:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829368.679743:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829368.679743:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829368.679744:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829368.679746:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829368.679747:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829368.679747:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829368.679748:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829368.679750:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829368.679750:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829368.679751:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829368.679752:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829368.679753:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829368.679756:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829368.679756:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829368.679758:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829368.679760:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787829368.679761:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829368.679762:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1787829368.679763:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829369.574730:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829369.574740:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829369.574805:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829369.574810:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829369.574814:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829370.598730:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829370.598737:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829370.598805:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829370.598810:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829370.598813:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829371.622732:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829371.622738:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829371.622813:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829371.622819:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829371.622822:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829372.646720:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829372.646729:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829372.646786:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829372.646790:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829372.646793:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829373.670737:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829373.670746:0:815:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1787829373.670748:0:815:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1787829373.670751:0:815:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829373.670756:0:815:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1787829373.670759:0:815:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829373.670760:0:815:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829373.670762:0:815:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829373.670829:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829373.670833:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829373.670837:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787829373.798711:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829373.798725:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1787829373.798729:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787829373.798734:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787829373.798740:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787829373.798750:0:40701:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 4174 00000100:00080000:0.0:1787829373.798763:0:40701:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787829373.798766:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829373.798767:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787829373.798772:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787829373.798774:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787829373.798777:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:0.0:1787829373.798779:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:0.0:1787829373.798781:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829373.798782:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:0.0:1787829373.798786:0:40701:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787829373.798788:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829373.798791:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787829373.798794:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787829373.798797:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829373.798799:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1787829373.798801:0:40701:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829373.798814:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787829373.798815:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829373.798819:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787829373.798821:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787829373.798826:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829373.798839:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787829373.798849:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d27c00 x1874670646823296/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787829373.798859:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787829373.798863:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829373.798884:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787829373.798886:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829373.798889:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1787829373.798890:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787829373.798892:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787829373.798894:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787829373.798896:0:40701:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4174 00000100:00080000:0.0:1787829373.798899:0:40701:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787829373.798900:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829373.798901:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787829373.798903:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787829373.798904:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787829373.798906:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000020:00000001:0.0:1787829373.798908:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:0.0:1787829373.798909:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829373.798910:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:0.0:1787829373.798912:0:40701:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787829373.798913:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829373.798914:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787829373.798920:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787829373.798921:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829373.798922:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1787829373.798926:0:40701:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1787829373.798929:0:40701:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829373.798930:0:40701:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829373.798932:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787829373.798934:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829373.798935:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787829373.798936:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787829373.798938:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829373.798941:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787829373.798946:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d252c0 x1874670646823424/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787829373.798955:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787829373.798957:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829373.798960:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787829373.798963:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1787829373.798963:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00080000:0.0:1787829373.798970:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1787829373.798970:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787829373.798974:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:3.0:1787829373.798975:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00080000:0.0:1787829373.798979:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787829373.798979:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787829373.798982:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829373.798988:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787829373.798990:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787829373.798991:0:40701:0:(import.c:485:import_select_connection()) Process entered 02000000:00000001:3.0:1787829373.798992:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787829373.798997:0:40701:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4174 00000100:00100000:3.0:1787829373.799001:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d27c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646823296:192.168.203.143@tcp:250:kworker.0 00000100:00080000:0.0:1787829373.799003:0:40701:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787829373.799008:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829373.799009:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:0.0:1787829373.799010:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787829373.799014:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787829373.799015:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 02000000:00000001:3.0:1787829373.799016:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000020:00000001:0.0:1787829373.799017:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000020:00000001:0.0:1787829373.799020:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 02000000:00000001:3.0:1787829373.799021:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829373.799022:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829373.799025:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 02000000:00000001:3.0:1787829373.799025:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00080000:0.0:1787829373.799030:0:40701:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 02000000:00000001:3.0:1787829373.799033:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829373.799034:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829373.799039:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787829373.799041:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787829373.799043:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829373.799044:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1787829373.799047:0:40701:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1787829373.799049:0:40701:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829373.799050:0:40701:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829373.799055:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:3.0:1787829373.799055:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 02000000:00000001:0.0:1787829373.799058:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829373.799059:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601351680 : -126234108199936 : ffff8d30d4d27c00) 00000100:00000001:0.0:1787829373.799060:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787829373.799064:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:3.0:1787829373.799065:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 02000000:00000001:0.0:1787829373.799066:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829373.799073:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787829373.799077:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d25680 x1874670646823552/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787829373.799083:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787829373.799084:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829373.799087:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787829373.799089:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829373.799092:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1787829373.799093:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787829373.799094:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787829373.799097:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787829373.799099:0:40701:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4174 00000100:00080000:0.0:1787829373.799104:0:40701:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787829373.799105:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829373.799106:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787829373.799107:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787829373.799108:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787829373.799110:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000020:00000001:0.0:1787829373.799111:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:0.0:1787829373.799113:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829373.799115:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000800:00000100:3.0:1787829373.799117:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00080000:0.0:1787829373.799120:0:40701:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787829373.799123:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829373.799124:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787829373.799126:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787829373.799128:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829373.799130:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1787829373.799131:0:40701:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1787829373.799133:0:40701:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829373.799134:0:40701:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829373.799137:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787829373.799139:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829373.799139:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1787829373.799140:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787829373.799144:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787829373.799145:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829373.799148:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787829373.799149:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829373.799154:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787829373.799155:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d26940 x1874670646823680/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787829373.799161:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787829373.799165:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787829373.799166:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829373.799168:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829373.799172:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829373.799179:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4298848000 00000100:00000001:3.0:1787829373.799193:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 00000100:00000001:3.0:1787829373.799202:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4298848000 00000100:00000001:3.0:1787829373.799208:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 00000100:00000001:3.0:1787829373.799212:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829373.799215:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829373.799218:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1787829373.799225:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829373/real 1787829373] req@ffff8d30d4d27c00 x1874670646823296/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829398 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1787829373.799237:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787829373.799239:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829373.799243:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829373.799246:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829373.799248:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829373.799251:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829373.799254:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829373.799259:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d27c00 x1874670646823296/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829398 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829373.799267:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829373.799271:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829373.799274:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829373.799275:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829373.799276:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829373.799280:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829373.799282:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829373.799285:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829373.799290:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829373.799293:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829373.799298:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d27c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646823296:192.168.203.143@tcp:250:kworker.0 00000100:00000001:3.0:1787829373.799302:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787829373.799304:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787829373.799306:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829373.799308:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787829373.799310:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829373.799312:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829373.799315:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d252c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646823424:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829373.799319:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1787829373.799321:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1787829373.799322:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829373.799324:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1787829373.799328:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829373.799334:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787829373.799336:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601341120 : -126234108210496 : ffff8d30d4d252c0) 00000100:00000001:3.0:1787829373.799341:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1787829373.799355:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:3.0:1787829373.799361:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787829373.799363:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829373.799365:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829373.799367:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1787829373.799370:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829373.799375:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298846742 00000100:00000001:3.0:1787829373.799378:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298848000 00000100:00000001:3.0:1787829373.799381:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298846742 00000100:00000001:3.0:1787829373.799383:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298848000 00000100:00000001:3.0:1787829373.799386:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829373.799388:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829373.799390:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1787829373.799395:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829373/real 1787829373] req@ffff8d30d4d252c0 x1874670646823424/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829398 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1787829373.799404:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787829373.799405:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829373.799408:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829373.799410:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829373.799412:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829373.799414:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829373.799416:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829373.799421:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d252c0 x1874670646823424/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829398 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829373.799428:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829373.799430:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829373.799433:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829373.799435:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829373.799436:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829373.799438:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829373.799440:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829373.799443:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829373.799446:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829373.799448:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829373.799452:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d252c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646823424:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829373.799457:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829373.799460:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829373.799462:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829373.799465:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829373.799468:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829373.799471:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829373.799473:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829373.799478:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829373.799480:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829373.799485:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829373.799486:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829373.799487:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829373.799488:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829373.799489:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829373.799491:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829373.799492:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829373.799492:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829373.799493:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829373.799494:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829373.799496:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829373.799497:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829373.799514:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829373.799522:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787829373.799523:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829373.799525:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1787829373.799526:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787829373.799527:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787829373.799528:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787829373.799529:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829373.799531:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787829373.799532:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829373.799534:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829373.799537:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d25680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646823552:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829373.799539:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1787829373.799541:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1787829373.799542:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829373.799544:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1787829373.799546:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829373.799550:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787829373.799551:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601342080 : -126234108209536 : ffff8d30d4d25680) 00000100:00000001:3.0:1787829373.799554:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1787829373.799564:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:3.0:1787829373.799568:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787829373.799569:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829373.799570:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829373.799571:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1787829373.799573:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829373.799576:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298846742 00000100:00000001:3.0:1787829373.799578:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298848000 00000100:00000001:3.0:1787829373.799580:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298846742 00000100:00000001:3.0:1787829373.799581:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298848000 00000100:00000001:3.0:1787829373.799583:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829373.799584:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829373.799585:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1787829373.799588:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829373/real 1787829373] req@ffff8d30d4d25680 x1874670646823552/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829398 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1787829373.799594:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787829373.799595:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829373.799597:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829373.799598:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829373.799599:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829373.799632:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829373.799633:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829373.799636:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d25680 x1874670646823552/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829398 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829373.799641:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829373.799643:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829373.799645:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829373.799646:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829373.799647:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829373.799648:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829373.799649:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829373.799650:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829373.799652:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829373.799654:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829373.799656:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d25680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646823552:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829373.799659:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787829373.799660:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787829373.799661:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829373.799663:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787829373.799664:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829373.799665:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829373.799667:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d26940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646823680:192.168.203.143@tcp:8:kworker.0 00000100:00000001:3.0:1787829373.799669:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1787829373.799670:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1787829373.799672:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829373.799673:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1787829373.799675:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829373.799679:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787829373.799680:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601346880 : -126234108204736 : ffff8d30d4d26940) 00000100:00000001:3.0:1787829373.799682:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1787829373.799690:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:3.0:1787829373.799693:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787829373.799694:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829373.799695:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829373.799696:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1787829373.799698:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829373.799700:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298846756 00000100:00000001:3.0:1787829373.799702:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298848000 00000100:00000001:3.0:1787829373.799704:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298846756 00000100:00000001:3.0:1787829373.799705:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298848000 00000100:00000001:3.0:1787829373.799707:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829373.799708:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829373.799709:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1787829373.799711:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829373/real 1787829373] req@ffff8d30d4d26940 x1874670646823680/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829398 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1787829373.799716:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787829373.799717:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829373.799718:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829373.799719:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829373.799720:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829373.799721:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829373.799723:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829373.799725:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d26940 x1874670646823680/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829398 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829373.799729:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829373.799731:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829373.799732:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829373.799734:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829373.799734:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829373.799736:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829373.799737:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829373.799739:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829373.799740:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829373.799741:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829373.799743:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d26940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646823680:192.168.203.143@tcp:8:kworker.0 00000100:00000001:3.0:1787829373.799745:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829373.799746:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829373.799747:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829373.799748:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829373.799749:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829373.799750:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829373.799751:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829373.799752:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829373.799753:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829373.799754:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829373.799755:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829373.799756:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829373.799757:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829373.799758:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829373.799759:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829373.799759:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829373.799760:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829373.799761:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829373.799762:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829373.799764:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829373.799765:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829373.799766:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829373.799768:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787829373.799769:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829373.799770:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1787829373.799771:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829374.694743:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829374.694753:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829374.694854:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829374.694858:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829374.694862:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829375.718743:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829375.718750:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829375.718823:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829375.718827:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829375.718830:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829376.742746:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829376.742754:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829376.742929:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829376.742934:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829376.742938:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829377.766765:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829377.766775:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829377.766840:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829377.766844:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829377.766849:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829378.790714:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829378.790723:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829378.790782:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829378.790789:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829378.790792:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787829378.918699:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829378.918707:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1787829378.918709:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787829378.918712:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787829378.918715:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787829378.918721:0:468:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 4179 00000100:00080000:2.0:1787829378.918731:0:468:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787829378.918733:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829378.918735:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787829378.918738:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787829378.918740:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787829378.918743:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:2.0:1787829378.918745:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:2.0:1787829378.918746:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829378.918747:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:2.0:1787829378.918751:0:468:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787829378.918753:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829378.918755:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787829378.918757:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787829378.918760:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829378.918761:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:2.0:1787829378.918763:0:468:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829378.918772:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787829378.918773:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829378.918776:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787829378.918778:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787829378.918781:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829378.918789:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787829378.918796:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2da40 x1874670646823808/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787829378.918803:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787829378.918806:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829378.918817:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787829378.918819:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829378.918822:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1787829378.918823:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787829378.918824:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787829378.918826:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787829378.918828:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4179 00000100:00080000:2.0:1787829378.918831:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787829378.918832:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829378.918833:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787829378.918835:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787829378.918836:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787829378.918838:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000020:00000001:2.0:1787829378.918839:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:2.0:1787829378.918840:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829378.918841:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:2.0:1787829378.918843:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787829378.918846:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829378.918847:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787829378.918851:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787829378.918853:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829378.918854:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:3.0:1787829378.918857:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000008:00100000:2.0:1787829378.918858:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1787829378.918862:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829378.918864:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829378.918865:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 02000000:00000001:2.0:1787829378.918867:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787829378.918870:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829378.918870:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1787829378.918871:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787829378.918873:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787829378.918876:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829378.918876:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787829378.918880:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829378.918881:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787829378.918886:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2f480 x1874670646823936/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:3.0:1787829378.918888:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787829378.918893:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787829378.918895:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787829378.918898:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829378.918899:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829378.918901:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787829378.918904:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829378.918908:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:3.0:1787829378.918909:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2da40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646823808:192.168.203.143@tcp:250:kworker.0 00000100:00000001:2.0:1787829378.918910:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787829378.918912:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787829378.918916:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:3.0:1787829378.918919:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00080000:2.0:1787829378.918920:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4179 00000100:00080000:2.0:1787829378.918925:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787829378.918926:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 02000000:00000001:3.0:1787829378.918927:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:2.0:1787829378.918928:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787829378.918931:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787829378.918933:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 02000000:00000001:3.0:1787829378.918933:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1787829378.918934:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000020:00000001:2.0:1787829378.918939:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:2.0:1787829378.918941:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 02000000:00000001:3.0:1787829378.918941:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:2.0:1787829378.918944:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:2.0:1787829378.918946:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787829378.918948:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829378.918949:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829378.918950:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787829378.918953:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787829378.918955:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829378.918956:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1787829378.918958:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1787829378.918960:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829378.918961:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829378.918965:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787829378.918966:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829378.918967:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787829378.918968:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787829378.918970:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:3.0:1787829378.918972:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417875008 : -126234291676608 : ffff8d30c9e2da40) 02000000:00000001:2.0:1787829378.918973:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829378.918977:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829378.918980:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00100000:2.0:1787829378.918981:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2c000 x1874670646824064/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787829378.918988:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787829378.918989:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829378.918991:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787829378.918993:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829378.918996:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1787829378.918997:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787829378.918999:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787829378.919000:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787829378.919002:0:468:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4179 00000100:00080000:2.0:1787829378.919004:0:468:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787829378.919006:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829378.919007:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787829378.919008:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787829378.919009:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787829378.919010:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000020:00000001:2.0:1787829378.919011:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:2.0:1787829378.919013:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829378.919014:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:2.0:1787829378.919018:0:468:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000800:00000100:3.0:1787829378.919022:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:2.0:1787829378.919023:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829378.919026:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787829378.919029:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787829378.919030:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829378.919031:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1787829378.919032:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000008:00000001:2.0:1787829378.919034:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829378.919035:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829378.919039:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787829378.919040:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829378.919040:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1787829378.919043:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787829378.919045:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787829378.919046:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829378.919048:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787829378.919050:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829378.919053:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787829378.919055:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2e940 x1874670646824192/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787829378.919058:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1787829378.919064:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787829378.919066:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829378.919067:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829378.919070:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829378.919074:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4298853120 00000100:00000001:3.0:1787829378.919085:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 00000100:00000001:3.0:1787829378.919093:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4298853120 00000100:00000001:3.0:1787829378.919098:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 00000100:00000001:3.0:1787829378.919102:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829378.919106:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829378.919108:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1787829378.919115:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829378/real 1787829378] req@ffff8d30c9e2da40 x1874670646823808/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829403 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1787829378.919126:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787829378.919128:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829378.919132:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829378.919135:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829378.919136:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829378.919139:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829378.919141:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829378.919146:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2da40 x1874670646823808/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829403 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829378.919154:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829378.919158:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829378.919161:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829378.919162:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829378.919164:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829378.919166:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829378.919168:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829378.919171:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829378.919175:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829378.919178:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829378.919181:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2da40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646823808:192.168.203.143@tcp:250:kworker.0 00000100:00000001:3.0:1787829378.919187:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829378.919189:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829378.919191:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829378.919194:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829378.919198:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829378.919200:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829378.919202:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829378.919206:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829378.919208:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829378.919212:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829378.919214:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829378.919216:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829378.919223:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787829378.919225:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829378.919227:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1787829378.919229:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787829378.919230:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787829378.919232:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787829378.919233:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829378.919235:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787829378.919237:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829378.919240:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829378.919244:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646823936:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829378.919247:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1787829378.919249:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1787829378.919251:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829378.919253:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1787829378.919256:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829378.919262:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787829378.919263:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417881728 : -126234291669888 : ffff8d30c9e2f480) 00000100:00000001:3.0:1787829378.919268:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1787829378.919281:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:3.0:1787829378.919286:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787829378.919288:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829378.919289:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829378.919291:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1787829378.919294:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829378.919298:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298851862 00000100:00000001:3.0:1787829378.919301:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298853120 00000100:00000001:3.0:1787829378.919304:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298851862 00000100:00000001:3.0:1787829378.919307:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298853120 00000100:00000001:3.0:1787829378.919309:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829378.919311:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829378.919313:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1787829378.919317:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829378/real 1787829378] req@ffff8d30c9e2f480 x1874670646823936/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829403 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1787829378.919327:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787829378.919328:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829378.919330:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829378.919332:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829378.919333:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829378.919335:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829378.919337:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829378.919341:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2f480 x1874670646823936/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829403 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829378.919348:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829378.919351:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829378.919353:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829378.919355:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829378.919356:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829378.919358:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829378.919360:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829378.919362:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829378.919364:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829378.919365:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829378.919368:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646823936:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829378.919370:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787829378.919371:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787829378.919373:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829378.919374:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787829378.919376:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829378.919377:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829378.919379:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646824064:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829378.919381:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1787829378.919382:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1787829378.919383:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829378.919384:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1787829378.919386:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829378.919390:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787829378.919391:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417868288 : -126234291683328 : ffff8d30c9e2c000) 00000100:00000001:3.0:1787829378.919394:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1787829378.919401:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:3.0:1787829378.919405:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787829378.919406:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829378.919407:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829378.919408:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1787829378.919410:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829378.919412:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298851862 00000100:00000001:3.0:1787829378.919414:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298853120 00000100:00000001:3.0:1787829378.919415:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298851862 00000100:00000001:3.0:1787829378.919416:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298853120 00000100:00000001:3.0:1787829378.919418:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829378.919420:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829378.919422:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1787829378.919426:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829378/real 1787829378] req@ffff8d30c9e2c000 x1874670646824064/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829403 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1787829378.919435:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787829378.919436:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829378.919439:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829378.919441:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829378.919442:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829378.919445:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829378.919446:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829378.919450:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2c000 x1874670646824064/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829403 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829378.919458:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829378.919460:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829378.919476:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829378.919478:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829378.919480:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829378.919482:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829378.919483:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829378.919487:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829378.919490:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829378.919492:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829378.919496:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646824064:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829378.919500:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787829378.919501:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787829378.919503:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829378.919505:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787829378.919507:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829378.919509:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829378.919512:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646824192:192.168.203.143@tcp:8:kworker.0 00000100:00000001:3.0:1787829378.919516:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1787829378.919518:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1787829378.919520:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829378.919522:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1787829378.919524:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829378.919530:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787829378.919532:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417878848 : -126234291672768 : ffff8d30c9e2e940) 00000100:00000001:3.0:1787829378.919535:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1787829378.919548:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:3.0:1787829378.919553:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787829378.919555:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829378.919557:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829378.919559:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1787829378.919561:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829378.919565:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298851875 00000100:00000001:3.0:1787829378.919567:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298853120 00000100:00000001:3.0:1787829378.919570:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298851875 00000100:00000001:3.0:1787829378.919572:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298853120 00000100:00000001:3.0:1787829378.919575:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829378.919577:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829378.919578:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1787829378.919583:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829378/real 1787829378] req@ffff8d30c9e2e940 x1874670646824192/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829403 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1787829378.919591:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787829378.919592:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829378.919594:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829378.919595:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829378.919597:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829378.919599:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829378.919630:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829378.919635:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2e940 x1874670646824192/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829403 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829378.919642:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829378.919645:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829378.919648:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829378.919649:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829378.919650:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829378.919652:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829378.919654:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829378.919656:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829378.919660:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829378.919662:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829378.919666:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646824192:192.168.203.143@tcp:8:kworker.0 00000100:00000001:3.0:1787829378.919669:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829378.919671:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829378.919673:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829378.919674:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829378.919676:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829378.919678:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829378.919679:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829378.919682:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829378.919683:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829378.919686:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829378.919687:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829378.919689:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829378.919690:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829378.919692:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829378.919693:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829378.919694:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829378.919696:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829378.919697:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829378.919698:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829378.919700:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829378.919702:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829378.919703:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829378.919704:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829378.919705:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829378.919707:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829378.919708:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829378.919709:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829378.919711:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829378.919712:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829378.919715:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829378.919716:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829378.919718:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829378.919721:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787829378.919723:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829378.919724:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1787829378.919726:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829379.814681:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829379.814686:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829379.814731:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829379.814735:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829379.814738:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829380.838696:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829380.838702:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829380.838757:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829380.838761:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829380.838763:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829381.862695:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829381.862702:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829381.862754:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829381.862759:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829381.862763:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829382.886712:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829382.886717:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829382.886837:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829382.886843:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829382.886849:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829383.910722:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829383.910730:0:815:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1787829383.910733:0:815:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1787829383.910737:0:815:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829383.910742:0:815:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1787829383.910744:0:815:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829383.910747:0:815:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829383.910750:0:815:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829383.910803:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829383.910807:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829383.910811:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787829384.038722:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829384.038730:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1787829384.038732:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787829384.038741:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787829384.038744:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787829384.038751:0:468:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 4184 00000100:00080000:3.0:1787829384.038760:0:468:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787829384.038762:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829384.038763:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:3.0:1787829384.038766:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1787829384.038768:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1787829384.038771:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:3.0:1787829384.038773:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:3.0:1787829384.038774:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829384.038775:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:3.0:1787829384.038778:0:468:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:3.0:1787829384.038780:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829384.038783:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787829384.038785:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829384.038787:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829384.038789:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:3.0:1787829384.038791:0:468:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829384.038800:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787829384.038801:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829384.038804:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787829384.038806:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787829384.038809:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829384.038818:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829384.038830:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c7016d00 x1874670646824320/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787829384.038838:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787829384.038841:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829384.038855:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787829384.038857:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829384.038859:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1787829384.038860:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787829384.038862:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787829384.038864:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787829384.038866:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4184 00000100:00080000:3.0:1787829384.038869:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787829384.038870:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829384.038872:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:3.0:1787829384.038874:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1787829384.038875:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1787829384.038878:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000020:00000001:3.0:1787829384.038880:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:3.0:1787829384.038882:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829384.038883:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:3.0:1787829384.038887:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:3.0:1787829384.038890:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829384.038892:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787829384.038897:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829384.038900:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829384.038902:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:3.0:1787829384.038906:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1787829384.038909:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829384.038912:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829384.038917:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787829384.038919:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829384.038921:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787829384.038922:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787829384.038925:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829384.038930:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829384.038936:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c70170c0 x1874670646824448/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787829384.038944:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787829384.038948:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829384.038957:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1787829384.038959:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787829384.038965:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1787829384.038966:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1787829384.038971:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00080000:3.0:1787829384.038973:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:1.0:1787829384.038974:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787829384.038979:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829384.038980:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787829384.038986:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 02000000:00000001:1.0:1787829384.038987:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787829384.038994:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829384.038995:0:468:0:(import.c:485:import_select_connection()) Process entered 02000000:00000001:1.0:1787829384.039001:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787829384.039014:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4184 00000100:00100000:1.0:1787829384.039015:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c7016d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646824320:192.168.203.143@tcp:250:kworker.0 00000100:00080000:3.0:1787829384.039021:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1787829384.039023:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:3.0:1787829384.039026:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829384.039028:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 02000000:00000001:1.0:1787829384.039030:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000020:00000001:3.0:1787829384.039031:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1787829384.039038:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 02000000:00000001:1.0:1787829384.039040:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829384.039042:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 02000000:00000001:1.0:1787829384.039044:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000020:00000001:3.0:1787829384.039046:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:3.0:1787829384.039049:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829384.039050:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:3.0:1787829384.039054:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 02000000:00000001:1.0:1787829384.039055:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829384.039062:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829384.039064:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787829384.039067:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829384.039069:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829384.039070:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:3.0:1787829384.039073:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1787829384.039075:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829384.039076:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829384.039082:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787829384.039083:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829384.039084:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787829384.039085:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:1.0:1787829384.039088:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 02000000:00000001:3.0:1787829384.039089:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829384.039094:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369547008 : -126234340004608 : ffff8d30c7016d00) 00000100:00000001:3.0:1787829384.039097:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829384.039103:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00100000:3.0:1787829384.039105:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c7017840 x1874670646824576/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787829384.039114:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787829384.039116:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829384.039119:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787829384.039121:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829384.039124:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1787829384.039126:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787829384.039128:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787829384.039129:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787829384.039131:0:468:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4184 00000100:00080000:3.0:1787829384.039134:0:468:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787829384.039135:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829384.039136:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:3.0:1787829384.039138:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1787829384.039139:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1787829384.039140:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000020:00000001:3.0:1787829384.039142:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:3.0:1787829384.039143:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829384.039149:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:3.0:1787829384.039152:0:468:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:3.0:1787829384.039155:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829384.039157:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787829384.039160:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829384.039162:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829384.039164:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:3.0:1787829384.039167:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1787829384.039169:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:1.0:1787829384.039175:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:3.0:1787829384.039176:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829384.039185:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787829384.039188:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829384.039191:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787829384.039192:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:1.0:1787829384.039197:0:2390:0:(events.c:34:request_out_callback()) Process entered 02000000:00000001:3.0:1787829384.039197:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829384.039207:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829384.039208:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829384.039213:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829384.039218:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c7014000 x1874670646824704/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787829384.039219:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787829384.039228:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829384.039240:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4298858240 00000100:00000001:3.0:1787829384.039241:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787829384.039247:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829384.039250:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829384.039260:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 00000100:00000001:1.0:1787829384.039272:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4298858240 00000100:00000001:1.0:1787829384.039279:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 00000100:00000001:1.0:1787829384.039284:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829384.039288:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829384.039291:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787829384.039300:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829384/real 1787829384] req@ffff8d30c7016d00 x1874670646824320/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829409 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787829384.039314:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787829384.039316:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787829384.039321:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787829384.039324:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787829384.039325:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829384.039328:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829384.039329:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787829384.039334:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c7016d00 x1874670646824320/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829409 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787829384.039340:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829384.039343:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787829384.039345:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829384.039347:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829384.039348:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829384.039351:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787829384.039352:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787829384.039355:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829384.039359:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787829384.039360:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787829384.039363:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c7016d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646824320:192.168.203.143@tcp:250:kworker.0 00000100:00000001:1.0:1787829384.039366:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787829384.039368:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787829384.039369:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829384.039370:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787829384.039372:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829384.039373:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829384.039376:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c70170c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646824448:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787829384.039378:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787829384.039380:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787829384.039381:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829384.039382:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787829384.039385:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829384.039390:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787829384.039392:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369547968 : -126234340003648 : ffff8d30c70170c0) 00000100:00000001:1.0:1787829384.039397:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1787829384.039412:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787829384.039417:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787829384.039420:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829384.039421:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829384.039423:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787829384.039426:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829384.039444:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298856982 00000100:00000001:1.0:1787829384.039447:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298858240 00000100:00000001:1.0:1787829384.039450:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298856982 00000100:00000001:1.0:1787829384.039454:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298858240 00000100:00000001:1.0:1787829384.039456:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829384.039459:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829384.039460:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787829384.039465:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829384/real 1787829384] req@ffff8d30c70170c0 x1874670646824448/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829409 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787829384.039473:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787829384.039474:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787829384.039476:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787829384.039477:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787829384.039478:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829384.039479:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829384.039480:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787829384.039483:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c70170c0 x1874670646824448/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829409 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787829384.039488:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829384.039490:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787829384.039492:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829384.039492:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829384.039493:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829384.039494:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787829384.039496:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787829384.039497:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829384.039499:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787829384.039500:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787829384.039502:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c70170c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646824448:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787829384.039505:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829384.039507:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829384.039508:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787829384.039511:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787829384.039514:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787829384.039517:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787829384.039519:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787829384.039523:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787829384.039525:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787829384.039527:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787829384.039528:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829384.039530:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829384.039530:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787829384.039531:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787829384.039533:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787829384.039534:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787829384.039534:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787829384.039535:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787829384.039536:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787829384.039537:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787829384.039538:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829384.039539:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829384.039547:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787829384.039548:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829384.039550:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787829384.039551:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1787829384.039552:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787829384.039553:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787829384.039554:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829384.039555:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787829384.039557:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829384.039558:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829384.039560:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c7017840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646824576:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787829384.039563:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787829384.039564:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787829384.039566:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829384.039567:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787829384.039569:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829384.039572:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787829384.039573:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369549888 : -126234340001728 : ffff8d30c7017840) 00000100:00000001:1.0:1787829384.039576:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1787829384.039585:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787829384.039587:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787829384.039589:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829384.039590:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829384.039592:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787829384.039593:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829384.039596:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298856982 00000100:00000001:1.0:1787829384.039598:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298858240 00000100:00000001:1.0:1787829384.039635:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298856983 00000100:00000001:1.0:1787829384.039637:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298858240 00000100:00000001:1.0:1787829384.039639:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829384.039641:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829384.039642:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787829384.039645:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829384/real 1787829384] req@ffff8d30c7017840 x1874670646824576/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829409 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787829384.039650:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787829384.039651:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787829384.039653:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787829384.039654:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787829384.039655:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829384.039656:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829384.039657:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787829384.039660:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c7017840 x1874670646824576/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829409 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787829384.039664:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829384.039665:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787829384.039667:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829384.039668:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829384.039668:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829384.039669:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787829384.039670:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787829384.039672:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829384.039675:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787829384.039676:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787829384.039679:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c7017840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646824576:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787829384.039681:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787829384.039683:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787829384.039684:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829384.039685:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787829384.039686:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829384.039687:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829384.039689:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c7014000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646824704:192.168.203.143@tcp:8:kworker.0 00000100:00000001:1.0:1787829384.039691:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787829384.039692:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787829384.039694:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829384.039695:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787829384.039697:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829384.039701:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787829384.039702:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369535488 : -126234340016128 : ffff8d30c7014000) 00000100:00000001:1.0:1787829384.039704:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1787829384.039711:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787829384.039714:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787829384.039715:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829384.039716:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829384.039717:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787829384.039719:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829384.039721:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298856996 00000100:00000001:1.0:1787829384.039723:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298858240 00000100:00000001:1.0:1787829384.039725:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298856996 00000100:00000001:1.0:1787829384.039727:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298858240 00000100:00000001:1.0:1787829384.039728:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829384.039729:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829384.039730:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787829384.039733:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829384/real 1787829384] req@ffff8d30c7014000 x1874670646824704/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829409 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787829384.039742:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787829384.039743:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787829384.039745:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787829384.039746:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787829384.039747:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829384.039748:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829384.039748:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787829384.039751:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c7014000 x1874670646824704/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829409 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787829384.039754:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829384.039756:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787829384.039757:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829384.039758:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829384.039760:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829384.039761:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787829384.039762:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787829384.039764:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829384.039766:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787829384.039767:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787829384.039769:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c7014000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646824704:192.168.203.143@tcp:8:kworker.0 00000100:00000001:1.0:1787829384.039771:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829384.039772:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829384.039773:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787829384.039774:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787829384.039775:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787829384.039776:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787829384.039776:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787829384.039778:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787829384.039779:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787829384.039780:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787829384.039780:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829384.039782:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829384.039783:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787829384.039784:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787829384.039785:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787829384.039786:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787829384.039787:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787829384.039788:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787829384.039788:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787829384.039790:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787829384.039790:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829384.039791:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829384.039793:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787829384.039794:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829384.039795:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787829384.039796:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829384.934701:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829384.934706:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829384.934758:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829384.934762:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829384.934765:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829385.958691:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829385.958696:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829385.958772:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829385.958776:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829385.958780:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829386.982716:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829386.982722:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829386.982807:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829386.982811:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829386.982813:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829388.006705:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829388.006710:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829388.006776:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829388.006782:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829388.006786:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829389.030709:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829389.030716:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829389.030780:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829389.030783:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829389.030786:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787829389.158694:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829389.158703:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1787829389.158706:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787829389.158709:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787829389.158712:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787829389.158719:0:40701:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 4189 00000100:00080000:0.0:1787829389.158728:0:40701:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787829389.158730:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829389.158732:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787829389.158735:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787829389.158737:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787829389.158740:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:0.0:1787829389.158742:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:0.0:1787829389.158743:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829389.158744:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:0.0:1787829389.158748:0:40701:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787829389.158750:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829389.158752:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787829389.158755:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787829389.158757:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829389.158759:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1787829389.158761:0:40701:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829389.158770:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787829389.158772:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829389.158774:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787829389.158776:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787829389.158780:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829389.158790:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787829389.158797:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d26940 x1874670646824832/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787829389.158805:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787829389.158808:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829389.158819:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787829389.158823:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829389.158826:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1787829389.158828:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787829389.158829:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787829389.158832:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787829389.158835:0:40701:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4189 00000100:00080000:0.0:1787829389.158839:0:40701:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787829389.158841:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829389.158843:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787829389.158845:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787829389.158846:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787829389.158849:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000020:00000001:0.0:1787829389.158851:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:0.0:1787829389.158854:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829389.158856:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:1.0:1787829389.158857:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00080000:0.0:1787829389.158859:0:40701:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787829389.158861:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829389.158862:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 02000000:00000001:0.0:1787829389.158863:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1787829389.158864:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787829389.158866:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787829389.158868:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829389.158869:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787829389.158871:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829389.158873:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 02000000:00000001:1.0:1787829389.158873:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787829389.158875:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000008:00100000:0.0:1787829389.158877:0:40701:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 02000000:00000001:1.0:1787829389.158878:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:0.0:1787829389.158880:0:40701:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829389.158882:0:40701:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829389.158883:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d26940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646824832:192.168.203.143@tcp:250:kworker.0 02000000:00000001:0.0:1787829389.158887:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:1.0:1787829389.158887:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787829389.158888:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829389.158890:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787829389.158891:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1787829389.158892:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787829389.158893:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829389.158895:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829389.158897:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:0.0:1787829389.158898:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829389.158901:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787829389.158903:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d25680 x1874670646824960/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787829389.158910:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787829389.158912:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829389.158915:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829389.158915:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787829389.158917:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601346880 : -126234108204736 : ffff8d30d4d26940) 00000100:00080000:0.0:1787829389.158918:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1787829389.158920:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00080000:0.0:1787829389.158922:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1787829389.158923:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787829389.158925:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787829389.158927:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787829389.158929:0:40701:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4189 00000100:00080000:0.0:1787829389.158945:0:40701:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787829389.158947:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829389.158949:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787829389.158951:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000800:00000100:1.0:1787829389.158952:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000020:00000001:0.0:1787829389.158953:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787829389.158955:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000020:00000001:0.0:1787829389.158957:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:0.0:1787829389.158959:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829389.158960:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:0.0:1787829389.158963:0:40701:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:1.0:1787829389.158963:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1787829389.158965:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829389.158967:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1787829389.158967:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 02000000:00000001:0.0:1787829389.158970:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829389.158970:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829389.158972:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829389.158972:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787829389.158975:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1787829389.158976:0:40701:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00000001:1.0:1787829389.158977:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:0.0:1787829389.158979:0:40701:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829389.158981:0:40701:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829389.158981:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4298863360 02000000:00000001:0.0:1787829389.158984:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787829389.158986:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829389.158988:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787829389.158989:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:1.0:1787829389.158990:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 02000000:00000001:0.0:1787829389.158991:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829389.158995:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4298863360 00000100:00000001:0.0:1787829389.158996:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829389.158999:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 00000100:00100000:0.0:1787829389.159000:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d252c0 x1874670646825088/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787829389.159003:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829389.159006:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829389.159008:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787829389.159009:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00000001:0.0:1787829389.159010:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829389.159013:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829389.159013:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829389/real 1787829389] req@ffff8d30d4d26940 x1874670646824832/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829414 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829389.159016:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829389.159019:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000400:00000001:1.0:1787829389.159024:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787829389.159026:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787829389.159030:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:1.0:1787829389.159030:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00080000:0.0:1787829389.159032:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787829389.159032:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787829389.159034:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829389.159035:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787829389.159037:0:40701:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4189 00000100:00000001:1.0:1787829389.159037:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829389.159039:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:0.0:1787829389.159041:0:40701:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787829389.159043:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00100000:1.0:1787829389.159043:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d26940 x1874670646824832/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829414 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787829389.159045:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787829389.159047:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787829389.159049:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787829389.159051:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:1.0:1787829389.159051:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1787829389.159053:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:0.0:1787829389.159055:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787829389.159055:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787829389.159057:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:1.0:1787829389.159058:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:0.0:1787829389.159060:0:40701:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:1.0:1787829389.159060:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829389.159061:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829389.159063:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829389.159064:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1787829389.159064:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787829389.159066:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 02000000:00000001:0.0:1787829389.159067:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787829389.159069:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1787829389.159069:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00000001:0.0:1787829389.159071:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00080000:1.0:1787829389.159072:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000008:00100000:0.0:1787829389.159073:0:40701:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00000001:1.0:1787829389.159075:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000008:00000001:0.0:1787829389.159076:0:40701:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829389.159078:0:40701:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829389.159079:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d26940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646824832:192.168.203.143@tcp:250:kworker.0 02000000:00000001:0.0:1787829389.159082:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787829389.159083:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829389.159084:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829389.159085:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787829389.159086:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:1.0:1787829389.159087:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 02000000:00000001:0.0:1787829389.159088:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829389.159088:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787829389.159091:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 00000100:00000001:0.0:1787829389.159092:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829389.159094:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00100000:0.0:1787829389.159096:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d27c00 x1874670646825216/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000020:00000001:1.0:1787829389.159096:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787829389.159098:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787829389.159103:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 00000100:00000001:0.0:1787829389.159104:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787829389.159105:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829389.159107:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829389.159112:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787829389.159116:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787829389.159118:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829389.159120:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829389.159127:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787829389.159129:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829389.159132:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787829389.159133:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1787829389.159134:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787829389.159136:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787829389.159137:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829389.159139:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787829389.159141:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829389.159144:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829389.159147:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d25680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646824960:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787829389.159149:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787829389.159151:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787829389.159152:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829389.159153:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787829389.159155:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829389.159160:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787829389.159161:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601342080 : -126234108209536 : ffff8d30d4d25680) 00000100:00000001:1.0:1787829389.159164:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1787829389.159175:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787829389.159178:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787829389.159180:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829389.159182:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829389.159183:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787829389.159186:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829389.159190:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298862102 00000100:00000001:1.0:1787829389.159194:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298863360 00000100:00000001:1.0:1787829389.159196:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298862102 00000100:00000001:1.0:1787829389.159198:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298863360 00000100:00000001:1.0:1787829389.159200:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829389.159202:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829389.159204:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787829389.159209:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829389/real 1787829389] req@ffff8d30d4d25680 x1874670646824960/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829414 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787829389.159219:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787829389.159220:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787829389.159223:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787829389.159225:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787829389.159226:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829389.159228:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829389.159230:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787829389.159234:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d25680 x1874670646824960/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829414 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787829389.159240:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829389.159242:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787829389.159244:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829389.159245:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829389.159246:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829389.159247:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787829389.159248:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787829389.159249:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829389.159252:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787829389.159253:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787829389.159255:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d25680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646824960:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787829389.159258:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787829389.159259:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787829389.159260:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829389.159261:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787829389.159263:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829389.159264:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829389.159266:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d252c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646825088:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787829389.159268:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787829389.159269:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787829389.159270:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829389.159271:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787829389.159274:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829389.159277:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787829389.159278:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601341120 : -126234108210496 : ffff8d30d4d252c0) 00000100:00000001:1.0:1787829389.159281:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1787829389.159290:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787829389.159293:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787829389.159294:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829389.159295:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829389.159296:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787829389.159298:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829389.159300:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298862102 00000100:00000001:1.0:1787829389.159302:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298863360 00000100:00000001:1.0:1787829389.159304:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298862102 00000100:00000001:1.0:1787829389.159305:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298863360 00000100:00000001:1.0:1787829389.159307:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829389.159308:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829389.159309:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787829389.159311:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829389/real 1787829389] req@ffff8d30d4d252c0 x1874670646825088/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829414 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787829389.159316:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787829389.159317:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787829389.159318:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787829389.159319:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787829389.159320:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829389.159321:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829389.159322:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787829389.159325:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d252c0 x1874670646825088/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829414 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787829389.159330:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829389.159331:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787829389.159333:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829389.159333:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829389.159334:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829389.159335:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787829389.159336:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787829389.159338:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829389.159339:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787829389.159341:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787829389.159342:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d252c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646825088:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787829389.159345:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787829389.159345:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787829389.159346:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829389.159348:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787829389.159349:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829389.159350:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829389.159352:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d27c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646825216:192.168.203.143@tcp:8:kworker.0 00000100:00000001:1.0:1787829389.159354:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787829389.159355:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787829389.159356:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829389.159357:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787829389.159359:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829389.159362:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787829389.159363:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601351680 : -126234108199936 : ffff8d30d4d27c00) 00000100:00000001:1.0:1787829389.159365:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1787829389.159371:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787829389.159373:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787829389.159374:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829389.159375:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829389.159377:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787829389.159378:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829389.159380:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298862115 00000100:00000001:1.0:1787829389.159382:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298863360 00000100:00000001:1.0:1787829389.159383:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298862115 00000100:00000001:1.0:1787829389.159385:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298863360 00000100:00000001:1.0:1787829389.159386:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829389.159387:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829389.159388:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787829389.159405:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829389/real 1787829389] req@ffff8d30d4d27c00 x1874670646825216/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829414 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787829389.159411:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787829389.159412:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787829389.159413:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787829389.159415:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787829389.159416:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829389.159417:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829389.159418:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787829389.159420:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d27c00 x1874670646825216/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829414 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787829389.159425:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829389.159427:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787829389.159428:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829389.159429:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829389.159430:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829389.159431:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787829389.159432:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787829389.159434:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829389.159435:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787829389.159437:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787829389.159439:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d27c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646825216:192.168.203.143@tcp:8:kworker.0 00000100:00000001:1.0:1787829389.159442:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829389.159443:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829389.159444:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787829389.159445:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787829389.159447:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787829389.159448:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787829389.159448:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787829389.159450:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787829389.159451:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787829389.159453:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787829389.159454:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829389.159455:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829389.159456:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787829389.159456:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787829389.159457:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787829389.159458:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787829389.159459:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787829389.159460:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787829389.159461:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787829389.159462:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787829389.159462:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829389.159463:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829389.159464:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787829389.159465:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787829389.159465:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787829389.159466:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787829389.159467:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787829389.159467:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787829389.159468:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787829389.159470:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787829389.159471:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829389.159476:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829389.159479:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787829389.159480:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829389.159481:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787829389.159482:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829390.054702:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829390.054708:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829390.054765:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829390.054769:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829390.054772:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829391.078718:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829391.078726:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829391.078778:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829391.078783:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829391.078786:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829392.102677:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829392.102682:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829392.102734:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829392.102737:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829392.102739:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829393.126699:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829393.126704:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829393.126763:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829393.126768:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829393.126771:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829394.150683:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829394.150692:0:815:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1787829394.150694:0:815:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1787829394.150698:0:815:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829394.150702:0:815:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1787829394.150704:0:815:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829394.150707:0:815:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829394.150709:0:815:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829394.150764:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829394.150768:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829394.150770:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787829394.278703:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829394.278713:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1787829394.278716:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787829394.278720:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787829394.278724:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787829394.278732:0:468:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 4194 00000100:00080000:2.0:1787829394.278744:0:468:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787829394.278747:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829394.278749:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787829394.278754:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787829394.278756:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787829394.278759:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:2.0:1787829394.278761:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:2.0:1787829394.278762:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829394.278763:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:2.0:1787829394.278767:0:468:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787829394.278769:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829394.278771:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787829394.278774:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787829394.278776:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829394.278778:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:2.0:1787829394.278780:0:468:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829394.278790:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787829394.278791:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829394.278794:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787829394.278797:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787829394.278800:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829394.278809:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787829394.278816:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2ed00 x1874670646825344/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787829394.278824:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787829394.278828:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829394.278841:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787829394.278843:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829394.278845:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1787829394.278847:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787829394.278848:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787829394.278849:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787829394.278851:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4194 00000100:00080000:2.0:1787829394.278854:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787829394.278855:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829394.278856:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787829394.278858:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787829394.278860:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787829394.278861:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000020:00000001:2.0:1787829394.278863:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:2.0:1787829394.278864:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829394.278865:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:2.0:1787829394.278867:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787829394.278868:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829394.278870:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787829394.278874:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829394.278876:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:2.0:1787829394.278876:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829394.278880:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:1.0:1787829394.278881:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000008:00100000:2.0:1787829394.278884:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00000001:1.0:1787829394.278885:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000008:00000001:2.0:1787829394.278889:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829394.278890:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787829394.278893:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829394.278894:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829394.278897:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1787829394.278899:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787829394.278900:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829394.278903:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787829394.278904:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1787829394.278908:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829394.278909:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787829394.278913:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829394.278914:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646825344:192.168.203.143@tcp:250:kworker.0 00000100:00000001:1.0:1787829394.278920:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:2.0:1787829394.278920:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829394.278924:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787829394.278927:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829394.278930:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00100000:2.0:1787829394.278930:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2f0c0 x1874670646825472/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:1.0:1787829394.278934:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829394.278944:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787829394.278948:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829394.278950:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787829394.278952:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417879808 : -126234291671808 : ffff8d30c9e2ed00) 00000100:00000001:2.0:1787829394.278952:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787829394.278959:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1787829394.278960:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00080000:2.0:1787829394.278964:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1787829394.278966:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787829394.278967:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787829394.278969:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787829394.278972:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4194 00000100:00080000:2.0:1787829394.278975:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787829394.278976:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829394.278977:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787829394.278979:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787829394.278981:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787829394.278982:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000020:00000001:2.0:1787829394.278984:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:2.0:1787829394.278985:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829394.278986:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:2.0:1787829394.278988:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787829394.278990:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829394.278991:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787829394.278994:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787829394.278995:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829394.278997:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1787829394.278999:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1787829394.279002:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:1.0:1787829394.279003:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:2.0:1787829394.279004:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829394.279010:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787829394.279011:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829394.279012:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787829394.279014:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787829394.279015:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829394.279017:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1787829394.279019:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829394.279023:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00100000:2.0:1787829394.279026:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2c780 x1874670646825600/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787829394.279028:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829394.279033:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1787829394.279038:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787829394.279039:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829394.279042:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829394.279042:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787829394.279047:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829394.279049:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1787829394.279050:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787829394.279051:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787829394.279055:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:1.0:1787829394.279056:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4298868480 00000100:00080000:2.0:1787829394.279057:0:468:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4194 00000100:00080000:2.0:1787829394.279062:0:468:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787829394.279064:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829394.279065:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787829394.279066:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:1.0:1787829394.279067:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 00000020:00000001:2.0:1787829394.279067:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787829394.279069:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000020:00000001:2.0:1787829394.279072:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:2.0:1787829394.279074:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787829394.279076:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4298868480 00000100:00000001:2.0:1787829394.279077:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:1.0:1787829394.279080:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 00000100:00000001:1.0:1787829394.279085:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00080000:2.0:1787829394.279086:0:468:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:1.0:1787829394.279089:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829394.279092:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829394.279093:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 02000000:00000001:2.0:1787829394.279098:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00100000:1.0:1787829394.279102:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829394/real 1787829394] req@ffff8d30c9e2ed00 x1874670646825344/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829419 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1787829394.279104:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787829394.279109:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829394.279114:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000400:00000001:1.0:1787829394.279119:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000008:00100000:2.0:1787829394.279119:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000400:00000001:1.0:1787829394.279122:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000008:00000001:2.0:1787829394.279125:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829394.279127:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1787829394.279128:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829394.279132:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 02000000:00000001:2.0:1787829394.279134:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:1.0:1787829394.279135:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:2.0:1787829394.279137:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829394.279139:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787829394.279140:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787829394.279141:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787829394.279143:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829394.279146:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829394.279147:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829394.279152:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2ed00 x1874670646825344/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829419 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829394.279155:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2fc00 x1874670646825728/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787829394.279174:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829394.279176:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787829394.279179:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829394.279181:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829394.279182:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787829394.279186:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829394.279188:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829394.279189:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829394.279191:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787829394.279192:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787829394.279194:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829394.279198:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787829394.279200:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787829394.279204:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646825344:192.168.203.143@tcp:250:kworker.0 00000100:00000001:1.0:1787829394.279207:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829394.279209:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829394.279210:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787829394.279212:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787829394.279214:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787829394.279216:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787829394.279217:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787829394.279220:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787829394.279221:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787829394.279225:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787829394.279226:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829394.279228:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829394.279233:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787829394.279235:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829394.279236:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787829394.279237:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1787829394.279238:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787829394.279239:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787829394.279240:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829394.279242:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787829394.279244:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829394.279245:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829394.279247:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646825472:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787829394.279250:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787829394.279251:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787829394.279252:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829394.279253:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787829394.279255:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829394.279259:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787829394.279260:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417880768 : -126234291670848 : ffff8d30c9e2f0c0) 00000100:00000001:1.0:1787829394.279263:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1787829394.279273:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787829394.279276:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787829394.279278:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829394.279279:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829394.279280:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787829394.279282:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829394.279285:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298867222 00000100:00000001:1.0:1787829394.279287:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298868480 00000100:00000001:1.0:1787829394.279289:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298867222 00000100:00000001:1.0:1787829394.279290:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298868480 00000100:00000001:1.0:1787829394.279291:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829394.279293:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829394.279294:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787829394.279297:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829394/real 1787829394] req@ffff8d30c9e2f0c0 x1874670646825472/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829419 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787829394.279302:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787829394.279303:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787829394.279304:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787829394.279305:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787829394.279306:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829394.279307:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829394.279308:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787829394.279311:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2f0c0 x1874670646825472/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829419 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787829394.279316:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829394.279317:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787829394.279319:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829394.279320:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829394.279320:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829394.279321:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787829394.279322:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787829394.279324:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829394.279326:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787829394.279327:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787829394.279329:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646825472:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787829394.279332:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787829394.279334:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787829394.279335:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829394.279337:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787829394.279339:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829394.279341:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829394.279344:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646825600:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787829394.279347:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787829394.279349:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787829394.279351:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829394.279352:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787829394.279366:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829394.279372:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787829394.279373:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417870208 : -126234291681408 : ffff8d30c9e2c780) 00000100:00000001:1.0:1787829394.279376:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1787829394.279390:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787829394.279394:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787829394.279397:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829394.279399:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829394.279401:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787829394.279404:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829394.279415:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298867222 00000100:00000001:1.0:1787829394.279419:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298868480 00000100:00000001:1.0:1787829394.279421:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298867222 00000100:00000001:1.0:1787829394.279422:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298868480 00000100:00000001:1.0:1787829394.279424:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829394.279425:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829394.279426:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787829394.279430:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829394/real 1787829394] req@ffff8d30c9e2c780 x1874670646825600/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829419 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787829394.279435:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787829394.279436:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787829394.279438:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787829394.279439:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787829394.279440:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829394.279442:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829394.279443:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787829394.279445:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2c780 x1874670646825600/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829419 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787829394.279450:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829394.279451:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787829394.279452:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829394.279453:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829394.279454:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829394.279455:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787829394.279456:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787829394.279458:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829394.279460:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787829394.279462:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787829394.279464:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646825600:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787829394.279467:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787829394.279468:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787829394.279469:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829394.279470:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787829394.279471:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829394.279473:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829394.279474:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646825728:192.168.203.143@tcp:8:kworker.0 00000100:00000001:1.0:1787829394.279476:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787829394.279478:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787829394.279479:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829394.279480:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787829394.279482:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829394.279486:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787829394.279487:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417883648 : -126234291667968 : ffff8d30c9e2fc00) 00000100:00000001:1.0:1787829394.279490:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1787829394.279498:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787829394.279501:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787829394.279503:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829394.279504:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829394.279505:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787829394.279506:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829394.279515:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298867235 00000100:00000001:1.0:1787829394.279517:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298868480 00000100:00000001:1.0:1787829394.279518:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298867235 00000100:00000001:1.0:1787829394.279520:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298868480 00000100:00000001:1.0:1787829394.279521:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829394.279522:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829394.279523:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787829394.279526:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829394/real 1787829394] req@ffff8d30c9e2fc00 x1874670646825728/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829419 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787829394.279535:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787829394.279536:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787829394.279538:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787829394.279539:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787829394.279540:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829394.279541:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829394.279542:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787829394.279544:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2fc00 x1874670646825728/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829419 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787829394.279548:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829394.279550:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787829394.279551:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829394.279552:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829394.279553:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829394.279554:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787829394.279555:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787829394.279556:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829394.279560:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787829394.279562:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787829394.279565:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646825728:192.168.203.143@tcp:8:kworker.0 00000100:00000001:1.0:1787829394.279569:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829394.279571:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829394.279573:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787829394.279574:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787829394.279576:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787829394.279578:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787829394.279579:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787829394.279581:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787829394.279583:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787829394.279585:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787829394.279586:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829394.279588:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829394.279589:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787829394.279591:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787829394.279593:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787829394.279594:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787829394.279595:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787829394.279597:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787829394.279598:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787829394.279630:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787829394.279632:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829394.279634:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829394.279635:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787829394.279637:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787829394.279640:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787829394.279641:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787829394.279643:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787829394.279644:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787829394.279646:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787829394.279648:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787829394.279649:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829394.279651:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829394.279655:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787829394.279656:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829394.279658:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787829394.279660:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829395.174700:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829395.174705:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829395.174763:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829395.174767:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829395.174770:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829396.198677:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829396.198682:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829396.198756:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829396.198760:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829396.198763:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829397.222686:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829397.222693:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829397.222751:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829397.222755:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829397.222758:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829398.246709:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829398.246716:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829398.246775:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829398.246779:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829398.246783:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829399.270695:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829399.270700:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829399.270754:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829399.270760:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829399.270764:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787829399.398669:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829399.398678:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1787829399.398680:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787829399.398683:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787829399.398687:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787829399.398693:0:468:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 4199 00000100:00080000:3.0:1787829399.398703:0:468:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787829399.398705:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829399.398706:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:3.0:1787829399.398710:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1787829399.398713:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1787829399.398715:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:3.0:1787829399.398717:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:3.0:1787829399.398718:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829399.398719:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:3.0:1787829399.398723:0:468:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:3.0:1787829399.398725:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829399.398728:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787829399.398730:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829399.398732:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829399.398734:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:3.0:1787829399.398736:0:468:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829399.398744:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787829399.398745:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829399.398747:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787829399.398749:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787829399.398752:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829399.398761:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829399.398769:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c7017c00 x1874670646825856/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787829399.398778:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787829399.398782:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829399.398796:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787829399.398800:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829399.398803:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1787829399.398804:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787829399.398805:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787829399.398808:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787829399.398811:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4199 00000100:00080000:3.0:1787829399.398814:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787829399.398815:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829399.398817:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:3.0:1787829399.398819:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1787829399.398821:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1787829399.398823:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:2.0:1787829399.398825:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:3.0:1787829399.398825:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:3.0:1787829399.398829:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829399.398831:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787829399.398831:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:2.0:1787829399.398836:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00080000:3.0:1787829399.398837:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787829399.398839:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787829399.398841:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829399.398843:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829399.398844:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787829399.398850:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787829399.398851:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787829399.398854:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829399.398855:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829399.398861:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829399.398861:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:3.0:1787829399.398867:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829399.398868:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c7017c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646825856:192.168.203.143@tcp:250:kworker.0 00000008:00000001:3.0:1787829399.398870:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829399.398876:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829399.398877:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787829399.398882:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1787829399.398883:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787829399.398886:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829399.398887:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829399.398893:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:3.0:1787829399.398893:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787829399.398897:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787829399.398899:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829399.398899:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829399.398904:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829399.398937:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787829399.398941:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369550848 : -126234340000768 : ffff8d30c7017c00) 00000100:00000001:2.0:1787829399.398946:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00100000:3.0:1787829399.398974:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c7016940 x1874670646825984/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787829399.398988:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:3.0:1787829399.398990:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787829399.398994:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829399.398997:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787829399.399000:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829399.399003:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1787829399.399005:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:2.0:1787829399.399006:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00080000:3.0:1787829399.399007:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787829399.399013:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829399.399016:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:2.0:1787829399.399017:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787829399.399021:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4199 00000100:00000001:2.0:1787829399.399023:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00080000:3.0:1787829399.399026:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787829399.399032:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829399.399038:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4298873600 00000100:00000001:2.0:1787829399.399047:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 00000100:00000001:2.0:1787829399.399054:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4298873600 00000100:00000001:2.0:1787829399.399058:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 00000100:00000001:2.0:1787829399.399062:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787829399.399065:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829399.399067:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1787829399.399074:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829399/real 1787829399] req@ffff8d30c7017c00 x1874670646825856/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829424 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1787829399.399086:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787829399.399088:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1787829399.399093:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1787829399.399096:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787829399.399098:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829399.399101:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787829399.399103:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787829399.399109:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c7017c00 x1874670646825856/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829424 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787829399.399118:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829399.399121:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787829399.399125:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829399.399127:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787829399.399128:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829399.399131:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787829399.399134:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787829399.399137:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829399.399142:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829399.399145:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829399.399146:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000001:3.0:1787829399.399151:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00100000:2.0:1787829399.399154:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c7017c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646825856:192.168.203.143@tcp:250:kworker.0 00000020:00000001:3.0:1787829399.399156:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1787829399.399160:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1787829399.399164:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:2.0:1787829399.399165:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:3.0:1787829399.399169:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:2.0:1787829399.399171:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829399.399174:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829399.399176:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 00000100:00000001:3.0:1787829399.399179:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 02000000:00000001:2.0:1787829399.399181:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1787829399.399187:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00080000:3.0:1787829399.399189:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000020:00000001:2.0:1787829399.399191:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:2.0:1787829399.399195:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 00000100:00000001:3.0:1787829399.399195:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829399.399201:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829399.399201:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787829399.399205:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1787829399.399208:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787829399.399210:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829399.399213:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829399.399220:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1787829399.399223:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829399.399226:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787829399.399228:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1787829399.399231:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1787829399.399233:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787829399.399234:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829399.399237:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787829399.399240:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829399.399240:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829399.399245:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829399.399247:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829399.399248:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:3.0:1787829399.399253:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1787829399.399257:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c7016940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646825984:192.168.203.143@tcp:38:kworker.0 00000008:00000001:3.0:1787829399.399257:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829399.399260:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829399.399266:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1787829399.399266:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787829399.399269:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829399.399270:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:3.0:1787829399.399272:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787829399.399275:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787829399.399276:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829399.399277:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829399.399282:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:3.0:1787829399.399285:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829399.399288:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829399.399292:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c7017c00 x1874670646826112/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787829399.399298:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787829399.399299:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787829399.399302:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369546048 : -126234340005568 : ffff8d30c7016940) 00000100:00000001:3.0:1787829399.399303:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829399.399308:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787829399.399311:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:2.0:1787829399.399312:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00080000:3.0:1787829399.399314:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1787829399.399335:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787829399.399336:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787829399.399339:0:468:0:(import.c:485:import_select_connection()) Process entered 00000800:00000100:2.0:1787829399.399343:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00080000:3.0:1787829399.399344:0:468:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4199 00000100:00000001:2.0:1787829399.399351:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00080000:3.0:1787829399.399352:0:468:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787829399.399356:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829399.399356:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829399.399358:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:2.0:1787829399.399359:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829399.399361:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:2.0:1787829399.399363:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000020:00000001:3.0:1787829399.399364:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:2.0:1787829399.399367:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829399.399367:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000020:00000001:3.0:1787829399.399369:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:2.0:1787829399.399373:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298872342 00000100:00000001:3.0:1787829399.399373:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829399.399374:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:2.0:1787829399.399379:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298873600 00000100:00080000:3.0:1787829399.399381:0:468:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787829399.399385:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298872342 00000100:00000001:3.0:1787829399.399386:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829399.399391:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298873600 02000000:00000001:3.0:1787829399.399392:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:2.0:1787829399.399396:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787829399.399399:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829399.399399:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787829399.399402:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 02000000:00000001:3.0:1787829399.399406:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787829399.399408:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829399/real 1787829399] req@ffff8d30c7016940 x1874670646825984/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829424 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829399.399410:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:3.0:1787829399.399414:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000400:00000001:2.0:1787829399.399421:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000008:00000001:3.0:1787829399.399422:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:2.0:1787829399.399424:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829399.399427:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829399.399428:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1787829399.399430:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787829399.399433:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829399.399434:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 02000000:00000001:3.0:1787829399.399436:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:2.0:1787829399.399437:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787829399.399442:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c7016940 x1874670646825984/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829424 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1787829399.399442:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829399.399448:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787829399.399450:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:2.0:1787829399.399452:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:3.0:1787829399.399454:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829399.399455:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787829399.399459:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829399.399460:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829399.399460:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829399.399461:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829399.399466:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787829399.399468:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787829399.399471:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:3.0:1787829399.399471:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c7014000 x1874670646826240/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829399.399476:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:2.0:1787829399.399477:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1787829399.399480:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c7016940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646825984:192.168.203.143@tcp:38:kworker.0 00000100:00000001:2.0:1787829399.399488:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829399.399492:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787829399.399494:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 00000100:00000001:3.0:1787829399.399494:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 02000000:00000001:2.0:1787829399.399498:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 00000100:00000001:3.0:1787829399.399499:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829399.399500:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00000001:3.0:1787829399.399502:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1787829399.399503:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:2.0:1787829399.399507:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:2.0:1787829399.399509:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787829399.399510:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1787829399.399512:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787829399.399513:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829399.399515:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829399.399518:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1787829399.399520:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829399.399521:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787829399.399523:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1787829399.399524:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1787829399.399525:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787829399.399526:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829399.399528:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787829399.399529:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787829399.399531:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787829399.399534:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c7017c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646826112:192.168.203.143@tcp:38:kworker.0 00000100:00000001:2.0:1787829399.399536:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787829399.399538:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1787829399.399539:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829399.399540:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1787829399.399542:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829399.399545:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787829399.399546:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369550848 : -126234340000768 : ffff8d30c7017c00) 00000100:00000001:2.0:1787829399.399549:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1787829399.399557:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:2.0:1787829399.399560:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1787829399.399562:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787829399.399563:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829399.399564:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1787829399.399565:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829399.399568:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298872342 00000100:00000001:2.0:1787829399.399569:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298873600 00000100:00000001:2.0:1787829399.399571:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298872342 00000100:00000001:2.0:1787829399.399573:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298873600 00000100:00000001:2.0:1787829399.399574:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787829399.399575:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829399.399576:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1787829399.399579:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829399/real 1787829399] req@ffff8d30c7017c00 x1874670646826112/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829424 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1787829399.399584:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787829399.399585:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1787829399.399586:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1787829399.399587:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787829399.399588:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829399.399590:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787829399.399591:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787829399.399593:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c7017c00 x1874670646826112/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829424 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787829399.399619:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829399.399622:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787829399.399624:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829399.399625:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787829399.399627:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829399.399629:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787829399.399630:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787829399.399632:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829399.399634:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:2.0:1787829399.399636:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1787829399.399638:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c7017c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646826112:192.168.203.143@tcp:38:kworker.0 00000100:00000001:2.0:1787829399.399640:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1787829399.399641:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787829399.399643:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829399.399644:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787829399.399646:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787829399.399648:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787829399.399651:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c7014000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646826240:192.168.203.143@tcp:8:kworker.0 00000100:00000001:2.0:1787829399.399653:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787829399.399655:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1787829399.399656:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829399.399658:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1787829399.399660:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829399.399664:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787829399.399665:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369535488 : -126234340016128 : ffff8d30c7014000) 00000100:00000001:2.0:1787829399.399668:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1787829399.399675:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:2.0:1787829399.399679:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1787829399.399680:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787829399.399681:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829399.399682:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1787829399.399684:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829399.399686:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298872356 00000100:00000001:2.0:1787829399.399688:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298873600 00000100:00000001:2.0:1787829399.399689:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298872356 00000100:00000001:2.0:1787829399.399691:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298873600 00000100:00000001:2.0:1787829399.399692:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787829399.399693:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829399.399694:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1787829399.399696:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829399/real 1787829399] req@ffff8d30c7014000 x1874670646826240/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829424 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1787829399.399701:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787829399.399702:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1787829399.399703:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1787829399.399705:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787829399.399705:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829399.399706:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787829399.399707:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787829399.399710:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c7014000 x1874670646826240/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829424 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787829399.399714:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829399.399715:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787829399.399716:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829399.399717:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787829399.399718:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829399.399719:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787829399.399720:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787829399.399722:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829399.399723:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:2.0:1787829399.399724:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1787829399.399727:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c7014000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646826240:192.168.203.143@tcp:8:kworker.0 00000100:00000001:2.0:1787829399.399729:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829399.399731:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787829399.399732:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787829399.399733:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1787829399.399734:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787829399.399735:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:2.0:1787829399.399735:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:2.0:1787829399.399737:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787829399.399737:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1787829399.399738:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787829399.399739:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829399.399740:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787829399.399741:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787829399.399742:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1787829399.399743:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787829399.399744:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:2.0:1787829399.399745:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:2.0:1787829399.399746:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787829399.399746:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1787829399.399747:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787829399.399748:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829399.399749:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829399.399751:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1787829399.399752:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829399.399753:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787829399.399754:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829400.294696:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829400.294701:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829400.294754:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829400.294759:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829400.294763:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829401.318694:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829401.318701:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829401.318784:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829401.318787:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829401.318790:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829402.342674:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829402.342681:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829402.342734:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829402.342739:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829402.342742:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829403.366677:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829403.366684:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829403.366734:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829403.366739:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829403.366742:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829404.390688:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829404.390697:0:815:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1787829404.390699:0:815:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1787829404.390702:0:815:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829404.390708:0:815:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1787829404.390710:0:815:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829404.390713:0:815:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829404.390716:0:815:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829404.390768:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829404.390772:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829404.390775:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787829404.518688:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829404.518698:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1787829404.518700:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787829404.518704:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787829404.518709:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787829404.518715:0:468:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 4205 00000100:00080000:3.0:1787829404.518726:0:468:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787829404.518729:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829404.518731:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:3.0:1787829404.518736:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1787829404.518738:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1787829404.518742:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:3.0:1787829404.518745:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:3.0:1787829404.518747:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829404.518749:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:3.0:1787829404.518754:0:468:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:3.0:1787829404.518757:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829404.518760:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787829404.518764:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829404.518767:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829404.518770:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:3.0:1787829404.518772:0:468:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829404.518782:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787829404.518785:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829404.518788:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787829404.518790:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787829404.518794:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829404.518805:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829404.518814:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c7017840 x1874670646826368/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787829404.518826:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787829404.518830:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829404.518843:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787829404.518845:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829404.518847:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1787829404.518849:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787829404.518850:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787829404.518852:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787829404.518854:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4205 00000100:00080000:3.0:1787829404.518857:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787829404.518858:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829404.518859:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:3.0:1787829404.518861:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1787829404.518862:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1787829404.518864:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000020:00000001:3.0:1787829404.518866:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:3.0:1787829404.518868:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829404.518869:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:3.0:1787829404.518871:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:3.0:1787829404.518873:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829404.518874:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787829404.518879:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829404.518880:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829404.518882:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:3.0:1787829404.518886:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00000001:2.0:1787829404.518887:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000008:00000001:3.0:1787829404.518889:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829404.518892:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787829404.518892:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829404.518896:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 02000000:00000001:3.0:1787829404.518898:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:2.0:1787829404.518899:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 02000000:00000001:3.0:1787829404.518901:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829404.518902:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829404.518904:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787829404.518906:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787829404.518908:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787829404.518908:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829404.518913:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829404.518915:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829404.518917:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829404.518922:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c70170c0 x1874670646826496/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787829404.518924:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c7017840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646826368:192.168.203.143@tcp:250:kworker.0 00000100:00000001:2.0:1787829404.518933:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:3.0:1787829404.518937:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 02000000:00000001:2.0:1787829404.518938:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:3.0:1787829404.518939:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829404.518942:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829404.518944:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829404.518946:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00080000:3.0:1787829404.518948:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 02000000:00000001:2.0:1787829404.518951:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787829404.518951:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1787829404.518952:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787829404.518955:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787829404.518958:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787829404.518960:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4205 00000100:00080000:3.0:1787829404.518963:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787829404.518965:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829404.518965:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:3.0:1787829404.518967:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:2.0:1787829404.518968:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000020:00000001:3.0:1787829404.518968:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1787829404.518969:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:2.0:1787829404.518972:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369549888 : -126234340001728 : ffff8d30c7017840) 00000020:00000001:3.0:1787829404.518973:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:3.0:1787829404.518978:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829404.518979:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:3.0:1787829404.518979:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:3.0:1787829404.518984:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:3.0:1787829404.518986:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829404.518987:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787829404.518990:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829404.518991:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829404.518992:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:3.0:1787829404.518994:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1787829404.518995:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829404.518996:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829404.518999:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787829404.519000:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829404.519002:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787829404.519003:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787829404.519004:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829404.519007:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829404.519010:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c7016d00 x1874670646826624/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787829404.519015:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787829404.519016:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829404.519017:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787829404.519019:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:2.0:1787829404.519024:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00080000:3.0:1787829404.519033:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1787829404.519034:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787829404.519036:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787829404.519037:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787829404.519039:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:2.0:1787829404.519043:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00080000:3.0:1787829404.519044:0:468:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4205 00000100:00000001:2.0:1787829404.519046:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787829404.519048:0:468:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787829404.519050:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1787829404.519051:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829404.519052:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:3.0:1787829404.519054:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1787829404.519055:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1787829404.519056:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000020:00000001:3.0:1787829404.519059:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:3.0:1787829404.519060:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829404.519061:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:2.0:1787829404.519064:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787829404.519064:0:468:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:3.0:1787829404.519066:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829404.519069:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:2.0:1787829404.519070:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4298878720 02000000:00000001:3.0:1787829404.519071:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829404.519076:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829404.519077:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:3.0:1787829404.519078:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1787829404.519080:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829404.519082:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 00000100:00000001:3.0:1787829404.519082:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829404.519090:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4298878720 02000000:00000001:3.0:1787829404.519090:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787829404.519093:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829404.519095:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787829404.519097:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:2.0:1787829404.519098:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 02000000:00000001:3.0:1787829404.519099:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829404.519105:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829404.519107:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829404.519108:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829404.519111:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1787829404.519113:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c7014b40 x1874670646826752/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787829404.519119:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829404/real 1787829404] req@ffff8d30c7017840 x1874670646826368/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829429 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829404.519124:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787829404.519127:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829404.519131:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:2.0:1787829404.519139:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787829404.519141:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1787829404.519144:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1787829404.519146:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787829404.519148:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829404.519150:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787829404.519152:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787829404.519155:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c7017840 x1874670646826368/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829429 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787829404.519161:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829404.519169:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787829404.519173:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829404.519178:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787829404.519180:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829404.519182:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787829404.519185:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787829404.519188:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829404.519193:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:2.0:1787829404.519196:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1787829404.519199:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c7017840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646826368:192.168.203.143@tcp:250:kworker.0 00000100:00000001:2.0:1787829404.519204:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829404.519207:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787829404.519208:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787829404.519210:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1787829404.519214:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787829404.519216:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:2.0:1787829404.519218:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:2.0:1787829404.519222:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787829404.519224:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1787829404.519227:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787829404.519228:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829404.519231:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829404.519238:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1787829404.519240:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829404.519242:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787829404.519244:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1787829404.519245:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1787829404.519247:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787829404.519248:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829404.519251:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787829404.519252:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787829404.519255:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787829404.519259:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c70170c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646826496:192.168.203.143@tcp:38:kworker.0 00000100:00000001:2.0:1787829404.519263:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787829404.519265:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1787829404.519267:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829404.519269:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1787829404.519272:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829404.519278:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787829404.519279:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369547968 : -126234340003648 : ffff8d30c70170c0) 00000100:00000001:2.0:1787829404.519284:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1787829404.519312:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:2.0:1787829404.519316:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1787829404.519318:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787829404.519319:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829404.519320:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1787829404.519322:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829404.519326:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298877462 00000100:00000001:2.0:1787829404.519328:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298878720 00000100:00000001:2.0:1787829404.519329:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298877462 00000100:00000001:2.0:1787829404.519331:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298878720 00000100:00000001:2.0:1787829404.519332:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787829404.519334:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829404.519335:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1787829404.519338:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829404/real 1787829404] req@ffff8d30c70170c0 x1874670646826496/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829429 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1787829404.519349:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787829404.519350:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1787829404.519352:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1787829404.519353:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787829404.519354:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829404.519355:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787829404.519356:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787829404.519359:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c70170c0 x1874670646826496/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829429 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787829404.519363:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829404.519365:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787829404.519366:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829404.519367:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787829404.519368:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829404.519369:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787829404.519370:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787829404.519372:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829404.519374:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:2.0:1787829404.519375:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1787829404.519377:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c70170c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646826496:192.168.203.143@tcp:38:kworker.0 00000100:00000001:2.0:1787829404.519380:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1787829404.519381:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787829404.519382:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829404.519383:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787829404.519384:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787829404.519386:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787829404.519388:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c7016d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646826624:192.168.203.143@tcp:38:kworker.0 00000100:00000001:2.0:1787829404.519389:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787829404.519391:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1787829404.519392:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829404.519393:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1787829404.519395:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829404.519398:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787829404.519399:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369547008 : -126234340004608 : ffff8d30c7016d00) 00000100:00000001:2.0:1787829404.519402:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1787829404.519409:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:2.0:1787829404.519411:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1787829404.519412:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787829404.519413:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829404.519414:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1787829404.519416:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829404.519418:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298877462 00000100:00000001:2.0:1787829404.519420:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298878720 00000100:00000001:2.0:1787829404.519421:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298877462 00000100:00000001:2.0:1787829404.519423:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298878720 00000100:00000001:2.0:1787829404.519424:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787829404.519425:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829404.519426:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1787829404.519428:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829404/real 1787829404] req@ffff8d30c7016d00 x1874670646826624/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829429 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1787829404.519433:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787829404.519434:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1787829404.519435:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1787829404.519436:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787829404.519437:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829404.519438:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787829404.519439:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787829404.519441:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c7016d00 x1874670646826624/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829429 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787829404.519445:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829404.519447:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787829404.519449:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829404.519450:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787829404.519450:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829404.519452:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787829404.519453:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787829404.519454:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829404.519456:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:2.0:1787829404.519457:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1787829404.519459:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c7016d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646826624:192.168.203.143@tcp:38:kworker.0 00000100:00000001:2.0:1787829404.519461:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1787829404.519462:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787829404.519463:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829404.519464:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787829404.519465:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787829404.519466:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787829404.519468:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c7014b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646826752:192.168.203.143@tcp:8:kworker.0 00000100:00000001:2.0:1787829404.519470:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787829404.519471:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1787829404.519472:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829404.519473:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1787829404.519474:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829404.519477:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787829404.519478:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369538368 : -126234340013248 : ffff8d30c7014b40) 00000100:00000001:2.0:1787829404.519480:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1787829404.519486:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:2.0:1787829404.519488:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1787829404.519489:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787829404.519490:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829404.519492:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1787829404.519493:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829404.519495:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298877475 00000100:00000001:2.0:1787829404.519497:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298878720 00000100:00000001:2.0:1787829404.519498:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298877475 00000100:00000001:2.0:1787829404.519500:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298878720 00000100:00000001:2.0:1787829404.519501:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787829404.519503:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829404.519504:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1787829404.519506:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829404/real 1787829404] req@ffff8d30c7014b40 x1874670646826752/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829429 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1787829404.519510:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787829404.519511:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1787829404.519512:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1787829404.519513:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787829404.519514:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829404.519515:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787829404.519516:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787829404.519518:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c7014b40 x1874670646826752/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829429 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787829404.519522:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829404.519523:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787829404.519525:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829404.519526:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787829404.519527:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829404.519528:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787829404.519529:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787829404.519530:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829404.519532:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:2.0:1787829404.519533:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1787829404.519535:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c7014b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646826752:192.168.203.143@tcp:8:kworker.0 00000100:00000001:2.0:1787829404.519537:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829404.519538:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787829404.519540:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787829404.519541:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1787829404.519542:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787829404.519543:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:2.0:1787829404.519544:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:2.0:1787829404.519546:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787829404.519547:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1787829404.519548:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787829404.519549:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829404.519550:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787829404.519551:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787829404.519552:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1787829404.519553:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787829404.519554:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:2.0:1787829404.519555:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:2.0:1787829404.519556:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787829404.519557:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1787829404.519559:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787829404.519560:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829404.519561:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787829404.519562:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787829404.519563:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1787829404.519565:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787829404.519566:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:2.0:1787829404.519568:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:2.0:1787829404.519569:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787829404.519570:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1787829404.519572:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787829404.519573:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829404.519575:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829404.519579:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1787829404.519580:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829404.519582:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787829404.519584:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829405.414665:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829405.414671:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829405.414717:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829405.414721:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829405.414724:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829406.438672:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829406.438680:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829406.438730:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829406.438733:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829406.438735:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829407.462709:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829407.462715:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829407.462786:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829407.462791:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829407.462800:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829408.486700:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829408.486705:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829408.486778:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829408.486784:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829408.486790:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829409.510680:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829409.510685:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829409.510760:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829409.510765:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829409.510769:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787829409.638738:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829409.638748:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1787829409.638751:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787829409.638755:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787829409.638760:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787829409.638768:0:468:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 4210 00000100:00080000:3.0:1787829409.638779:0:468:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787829409.638782:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829409.638784:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:3.0:1787829409.638788:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1787829409.638792:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1787829409.638795:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:3.0:1787829409.638799:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:3.0:1787829409.638801:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829409.638803:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:3.0:1787829409.638808:0:468:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:3.0:1787829409.638811:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829409.638815:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787829409.638819:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829409.638823:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829409.638826:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:3.0:1787829409.638830:0:468:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829409.638846:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787829409.638848:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829409.638859:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787829409.638862:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787829409.638867:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829409.638880:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829409.638889:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c70152c0 x1874670646826880/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787829409.638898:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787829409.638901:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829409.638917:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787829409.638920:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829409.638922:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1787829409.638924:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787829409.638926:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787829409.638929:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787829409.638932:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4210 00000100:00080000:3.0:1787829409.638937:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787829409.638939:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829409.638942:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:3.0:1787829409.638945:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1787829409.638946:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1787829409.638949:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000020:00000001:3.0:1787829409.638953:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:3.0:1787829409.638955:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829409.638957:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:3.0:1787829409.638963:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:3.0:1787829409.638979:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829409.638980:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:3.0:1787829409.638983:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:2.0:1787829409.638991:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1787829409.638995:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 02000000:00000001:3.0:1787829409.638995:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787829409.639003:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 02000000:00000001:3.0:1787829409.639003:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829409.639011:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:2.0:1787829409.639012:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000008:00100000:3.0:1787829409.639022:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 02000000:00000001:2.0:1787829409.639023:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000008:00000001:3.0:1787829409.639030:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829409.639031:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787829409.639038:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829409.639042:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829409.639050:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787829409.639052:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829409.639054:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787829409.639055:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00100000:2.0:1787829409.639057:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c70152c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646826880:192.168.203.143@tcp:250:kworker.0 02000000:00000001:3.0:1787829409.639057:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829409.639067:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:3.0:1787829409.639068:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829409.639074:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000100:00100000:3.0:1787829409.639077:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c7015680 x1874670646827008/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:2.0:1787829409.639079:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829409.639086:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1787829409.639095:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829409.639097:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787829409.639100:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829409.639104:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787829409.639106:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829409.639109:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1787829409.639111:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787829409.639113:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787829409.639114:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787829409.639116:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:2.0:1787829409.639121:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369540288 : -126234340011328 : ffff8d30c70152c0) 00000100:00080000:3.0:1787829409.639122:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4210 00000100:00080000:3.0:1787829409.639128:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787829409.639130:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:3.0:1787829409.639132:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829409.639134:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:3.0:1787829409.639136:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1787829409.639137:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1787829409.639139:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000020:00000001:3.0:1787829409.639141:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:3.0:1787829409.639143:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829409.639145:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:3.0:1787829409.639148:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:3.0:1787829409.639150:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829409.639152:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787829409.639155:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829409.639156:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829409.639158:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:3.0:1787829409.639160:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1787829409.639162:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829409.639163:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829409.639167:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787829409.639168:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829409.639169:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787829409.639170:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787829409.639172:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829409.639176:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:2.0:1787829409.639178:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00100000:3.0:1787829409.639186:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c70143c0 x1874670646827136/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787829409.639193:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787829409.639195:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787829409.639195:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829409.639204:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829409.639206:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00080000:3.0:1787829409.639206:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:2.0:1787829409.639210:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787829409.639211:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1787829409.639217:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1787829409.639218:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787829409.639227:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787829409.639229:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829409.639231:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:2.0:1787829409.639235:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4298883840 00000100:00080000:3.0:1787829409.639236:0:468:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4210 00000100:00080000:3.0:1787829409.639246:0:468:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787829409.639262:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829409.639264:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:2.0:1787829409.639266:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 00000020:00000001:3.0:1787829409.639267:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1787829409.639271:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1787829409.639274:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000020:00000001:3.0:1787829409.639277:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:2.0:1787829409.639278:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4298883840 00000100:00000001:3.0:1787829409.639281:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829409.639283:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:2.0:1787829409.639285:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 00000100:00080000:3.0:1787829409.639291:0:468:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787829409.639293:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787829409.639302:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829409.639304:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00000001:3.0:1787829409.639306:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829409.639309:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00100000:2.0:1787829409.639313:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829409/real 1787829409] req@ffff8d30c70152c0 x1874670646826880/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829434 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1787829409.639313:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829409.639318:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829409.639327:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:3.0:1787829409.639332:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1787829409.639338:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829409.639348:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829409.639358:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787829409.639360:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:2.0:1787829409.639361:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787829409.639364:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829409.639364:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787829409.639367:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:2.0:1787829409.639370:0:2390:0:(events.c:78:reply_in_callback()) Process entered 02000000:00000001:3.0:1787829409.639371:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829409.639376:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829409.639379:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829409.639380:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829409.639392:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787829409.639396:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829409.639397:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c7014f00 x1874670646827264/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787829409.639403:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c70152c0 x1874670646826880/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829434 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829409.639431:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787829409.639433:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829409.639435:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829409.639441:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829409.639445:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787829409.639448:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829409.639449:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787829409.639450:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829409.639452:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787829409.639455:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787829409.639460:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829409.639465:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:2.0:1787829409.639467:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1787829409.639470:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c70152c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646826880:192.168.203.143@tcp:250:kworker.0 00000100:00000001:2.0:1787829409.639479:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829409.639481:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787829409.639485:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787829409.639488:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1787829409.639492:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787829409.639494:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:2.0:1787829409.639496:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:2.0:1787829409.639498:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787829409.639500:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1787829409.639502:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787829409.639503:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829409.639506:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829409.639513:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1787829409.639515:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829409.639517:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787829409.639519:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1787829409.639521:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1787829409.639522:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787829409.639524:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829409.639526:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787829409.639528:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787829409.639531:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787829409.639535:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c7015680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646827008:192.168.203.143@tcp:38:kworker.0 00000100:00000001:2.0:1787829409.639539:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787829409.639541:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1787829409.639543:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829409.639545:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1787829409.639548:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829409.639555:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787829409.639556:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369541248 : -126234340010368 : ffff8d30c7015680) 00000100:00000001:2.0:1787829409.639560:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1787829409.639571:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:2.0:1787829409.639575:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1787829409.639577:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787829409.639578:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829409.639580:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1787829409.639582:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829409.639585:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298882582 00000100:00000001:2.0:1787829409.639586:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298883840 00000100:00000001:2.0:1787829409.639588:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298882582 00000100:00000001:2.0:1787829409.639590:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298883840 00000100:00000001:2.0:1787829409.639591:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787829409.639592:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829409.639594:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1787829409.639596:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829409/real 1787829409] req@ffff8d30c7015680 x1874670646827008/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829434 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1787829409.639642:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787829409.639643:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1787829409.639645:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1787829409.639646:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787829409.639647:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829409.639649:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787829409.639650:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787829409.639653:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c7015680 x1874670646827008/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829434 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787829409.639657:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829409.639659:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787829409.639661:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829409.639662:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787829409.639662:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829409.639663:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787829409.639665:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787829409.639666:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829409.639668:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:2.0:1787829409.639670:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1787829409.639672:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c7015680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646827008:192.168.203.143@tcp:38:kworker.0 00000100:00000001:2.0:1787829409.639675:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1787829409.639676:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787829409.639677:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829409.639679:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787829409.639680:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787829409.639681:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787829409.639683:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c70143c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646827136:192.168.203.143@tcp:38:kworker.0 00000100:00000001:2.0:1787829409.639686:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787829409.639687:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1787829409.639689:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829409.639690:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1787829409.639692:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829409.639695:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787829409.639696:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369536448 : -126234340015168 : ffff8d30c70143c0) 00000100:00000001:2.0:1787829409.639699:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1787829409.639706:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:2.0:1787829409.639711:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1787829409.639713:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787829409.639714:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829409.639716:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1787829409.639719:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829409.639722:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298882583 00000100:00000001:2.0:1787829409.639725:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298883840 00000100:00000001:2.0:1787829409.639727:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298882583 00000100:00000001:2.0:1787829409.639729:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298883840 00000100:00000001:2.0:1787829409.639732:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787829409.639734:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829409.639735:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1787829409.639740:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829409/real 1787829409] req@ffff8d30c70143c0 x1874670646827136/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829434 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1787829409.639749:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787829409.639751:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1787829409.639754:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1787829409.639755:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787829409.639757:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829409.639758:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787829409.639760:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787829409.639764:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c70143c0 x1874670646827136/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829434 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787829409.639768:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829409.639770:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787829409.639772:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829409.639773:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787829409.639774:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829409.639775:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787829409.639776:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787829409.639778:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829409.639780:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:2.0:1787829409.639781:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1787829409.639783:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c70143c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646827136:192.168.203.143@tcp:38:kworker.0 00000100:00000001:2.0:1787829409.639786:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1787829409.639787:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787829409.639788:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829409.639789:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787829409.639790:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787829409.639792:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787829409.639794:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c7014f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646827264:192.168.203.143@tcp:8:kworker.0 00000100:00000001:2.0:1787829409.639796:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787829409.639798:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1787829409.639799:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829409.639800:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1787829409.639802:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829409.639806:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787829409.639807:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369539328 : -126234340012288 : ffff8d30c7014f00) 00000100:00000001:2.0:1787829409.639810:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1787829409.639818:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:2.0:1787829409.639821:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1787829409.639822:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787829409.639823:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829409.639824:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1787829409.639826:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829409.639829:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298882596 00000100:00000001:2.0:1787829409.639830:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298883840 00000100:00000001:2.0:1787829409.639832:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298882596 00000100:00000001:2.0:1787829409.639834:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298883840 00000100:00000001:2.0:1787829409.639835:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787829409.639836:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829409.639838:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1787829409.639840:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829409/real 1787829409] req@ffff8d30c7014f00 x1874670646827264/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829434 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1787829409.639844:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787829409.639845:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1787829409.639846:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1787829409.639847:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787829409.639848:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829409.639849:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787829409.639850:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787829409.639853:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c7014f00 x1874670646827264/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829434 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787829409.639857:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829409.639858:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787829409.639859:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829409.639860:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787829409.639861:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829409.639862:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787829409.639863:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787829409.639865:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829409.639866:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:2.0:1787829409.639868:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1787829409.639869:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c7014f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646827264:192.168.203.143@tcp:8:kworker.0 00000100:00000001:2.0:1787829409.639872:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829409.639873:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787829409.639874:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787829409.639875:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1787829409.639877:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787829409.639878:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:2.0:1787829409.639879:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:2.0:1787829409.639880:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787829409.639881:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1787829409.639882:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787829409.639883:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829409.639884:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787829409.639885:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787829409.639886:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1787829409.639887:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787829409.639888:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:2.0:1787829409.639889:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:2.0:1787829409.639890:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787829409.639890:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1787829409.639891:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787829409.639892:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829409.639893:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787829409.639894:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787829409.639894:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1787829409.639895:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787829409.639896:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:2.0:1787829409.639905:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:2.0:1787829409.639906:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787829409.639907:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1787829409.639908:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787829409.639909:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829409.639913:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829409.639915:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1787829409.639916:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829409.639917:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787829409.639918:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829410.534706:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829410.534711:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829410.534793:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829410.534799:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829410.534803:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829411.558678:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829411.558684:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829411.558745:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829411.558748:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829411.558751:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829412.582706:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829412.582711:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829412.582758:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829412.582762:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829412.582764:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829413.606676:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829413.606683:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829413.606734:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829413.606738:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829413.606740:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829414.630742:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829414.630751:0:815:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1787829414.630754:0:815:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1787829414.630757:0:815:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829414.630762:0:815:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1787829414.630764:0:815:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829414.630766:0:815:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829414.630769:0:815:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829414.630825:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829414.630828:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829414.630830:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787829414.758740:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829414.758748:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1787829414.758750:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787829414.758753:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787829414.758757:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787829414.758764:0:468:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 4215 00000100:00080000:2.0:1787829414.758775:0:468:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787829414.758778:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829414.758780:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787829414.758785:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787829414.758788:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787829414.758791:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:2.0:1787829414.758794:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:2.0:1787829414.758797:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829414.758798:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:2.0:1787829414.758804:0:468:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787829414.758807:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829414.758811:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787829414.758813:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787829414.758815:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829414.758817:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:2.0:1787829414.758819:0:468:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829414.758828:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787829414.758829:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829414.758832:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787829414.758835:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787829414.758838:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829414.758847:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787829414.758854:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2de00 x1874670646827392/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787829414.758862:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787829414.758866:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829414.758879:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787829414.758881:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829414.758883:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1787829414.758884:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787829414.758886:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787829414.758888:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787829414.758889:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4215 00000100:00080000:2.0:1787829414.758893:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787829414.758894:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829414.758895:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787829414.758897:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787829414.758899:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787829414.758900:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000020:00000001:2.0:1787829414.758902:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:2.0:1787829414.758903:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829414.758905:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:2.0:1787829414.758907:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787829414.758909:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829414.758912:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:2.0:1787829414.758912:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1787829414.758917:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 02000000:00000001:2.0:1787829414.758918:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829414.758920:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 02000000:00000001:2.0:1787829414.758921:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829414.758924:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787829414.758924:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:1.0:1787829414.758927:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000008:00100000:2.0:1787829414.758928:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 02000000:00000001:1.0:1787829414.758932:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000008:00000001:2.0:1787829414.758932:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829414.758934:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829414.758936:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787829414.758939:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1787829414.758940:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829414.758941:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829414.758944:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787829414.758946:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00100000:1.0:1787829414.758947:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646827392:192.168.203.143@tcp:250:kworker.0 02000000:00000001:2.0:1787829414.758948:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829414.758955:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:2.0:1787829414.758957:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829414.758959:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787829414.758962:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829414.758965:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00100000:2.0:1787829414.758967:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2c3c0 x1874670646827520/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:1.0:1787829414.758970:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829414.758980:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787829414.758983:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829414.758985:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787829414.758987:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829414.758988:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417875968 : -126234291675648 : ffff8d30c9e2de00) 00000100:00000001:1.0:1787829414.758994:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00080000:2.0:1787829414.758994:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829414.759008:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1787829414.759009:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787829414.759010:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787829414.759013:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787829414.759015:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4215 00000100:00080000:2.0:1787829414.759019:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787829414.759021:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829414.759022:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787829414.759024:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787829414.759026:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000800:00000100:1.0:1787829414.759027:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000020:00000001:2.0:1787829414.759028:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000020:00000001:2.0:1787829414.759032:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:2.0:1787829414.759033:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829414.759035:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:1.0:1787829414.759038:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00080000:2.0:1787829414.759039:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787829414.759043:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829414.759044:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 02000000:00000001:2.0:1787829414.759045:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1787829414.759049:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829414.759050:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829414.759053:0:2390:0:(events.c:68:request_out_callback()) Process leaving 02000000:00000001:2.0:1787829414.759054:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829414.759056:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:1.0:1787829414.759059:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000008:00100000:2.0:1787829414.759059:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1787829414.759063:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829414.759066:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4298888960 00000100:00000001:2.0:1787829414.759066:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829414.759075:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787829414.759077:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829414.759078:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 00000100:00000001:2.0:1787829414.759080:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787829414.759083:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:1.0:1787829414.759086:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4298888960 02000000:00000001:2.0:1787829414.759086:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829414.759092:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 00000100:00000001:1.0:1787829414.759096:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787829414.759096:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829414.759102:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829414.759104:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1787829414.759107:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2d2c0 x1874670646827648/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787829414.759111:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829414/real 1787829414] req@ffff8d30c9e2de00 x1874670646827392/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829439 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787829414.759133:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787829414.759137:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829414.759141:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:1.0:1787829414.759145:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00080000:2.0:1787829414.759145:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000001:1.0:1787829414.759148:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00080000:2.0:1787829414.759150:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1787829414.759152:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1787829414.759153:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:1.0:1787829414.759156:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00080000:2.0:1787829414.759157:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787829414.759159:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829414.759160:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:1.0:1787829414.759163:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00080000:2.0:1787829414.759165:0:468:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4215 00000100:00080000:2.0:1787829414.759168:0:468:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787829414.759170:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829414.759171:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787829414.759172:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:1.0:1787829414.759175:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:2.0:1787829414.759175:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787829414.759177:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000020:00000001:2.0:1787829414.759179:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:2.0:1787829414.759182:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829414.759185:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00100000:1.0:1787829414.759186:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2de00 x1874670646827392/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829439 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829414.759189:0:468:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787829414.759195:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829414.759199:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1787829414.759202:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:2.0:1787829414.759203:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829414.759207:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 02000000:00000001:2.0:1787829414.759207:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829414.759211:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829414.759223:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829414.759224:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829414.759231:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787829414.759231:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:1.0:1787829414.759235:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000008:00100000:2.0:1787829414.759235:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000008:00000001:2.0:1787829414.759241:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1787829414.759242:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00000001:2.0:1787829414.759244:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1787829414.759247:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 02000000:00000001:2.0:1787829414.759249:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:1.0:1787829414.759251:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 02000000:00000001:2.0:1787829414.759252:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829414.759255:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787829414.759257:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00100000:1.0:1787829414.759258:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646827392:192.168.203.143@tcp:250:kworker.0 02000000:00000001:2.0:1787829414.759260:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829414.759266:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829414.759267:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829414.759271:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829414.759273:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 00000100:00100000:2.0:1787829414.759274:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2cb40 x1874670646827776/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:1.0:1787829414.759276:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787829414.759281:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00000001:2.0:1787829414.759283:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000020:00000001:1.0:1787829414.759284:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000100:00000001:2.0:1787829414.759286:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829414.759287:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 00000100:00000001:2.0:1787829414.759289:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829414.759292:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787829414.759294:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787829414.759299:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787829414.759301:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829414.759303:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829414.759309:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787829414.759311:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829414.759312:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787829414.759314:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1787829414.759315:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787829414.759316:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787829414.759317:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829414.759318:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787829414.759319:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829414.759321:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829414.759324:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646827520:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787829414.759327:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787829414.759328:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787829414.759329:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829414.759330:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787829414.759332:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829414.759337:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787829414.759338:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417869248 : -126234291682368 : ffff8d30c9e2c3c0) 00000100:00000001:1.0:1787829414.759341:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1787829414.759350:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787829414.759353:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787829414.759355:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829414.759356:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829414.759357:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787829414.759359:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829414.759362:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298887702 00000100:00000001:1.0:1787829414.759364:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298888960 00000100:00000001:1.0:1787829414.759365:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298887702 00000100:00000001:1.0:1787829414.759367:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298888960 00000100:00000001:1.0:1787829414.759368:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829414.759369:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829414.759370:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787829414.759373:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829414/real 1787829414] req@ffff8d30c9e2c3c0 x1874670646827520/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829439 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787829414.759383:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787829414.759384:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787829414.759386:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787829414.759387:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787829414.759388:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829414.759389:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829414.759390:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787829414.759392:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2c3c0 x1874670646827520/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829439 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787829414.759396:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829414.759397:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787829414.759399:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829414.759400:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829414.759401:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829414.759402:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787829414.759403:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787829414.759405:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829414.759407:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787829414.759408:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787829414.759410:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646827520:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787829414.759412:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787829414.759413:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787829414.759414:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829414.759415:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787829414.759416:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829414.759417:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829414.759419:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2d2c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646827648:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787829414.759421:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787829414.759421:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787829414.759423:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829414.759424:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787829414.759425:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829414.759428:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787829414.759429:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417873088 : -126234291678528 : ffff8d30c9e2d2c0) 00000100:00000001:1.0:1787829414.759431:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1787829414.759437:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787829414.759439:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787829414.759440:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829414.759441:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829414.759442:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787829414.759444:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829414.759445:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298887702 00000100:00000001:1.0:1787829414.759447:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298888960 00000100:00000001:1.0:1787829414.759449:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298887702 00000100:00000001:1.0:1787829414.759450:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298888960 00000100:00000001:1.0:1787829414.759452:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829414.759453:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829414.759454:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787829414.759456:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829414/real 1787829414] req@ffff8d30c9e2d2c0 x1874670646827648/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829439 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787829414.759460:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787829414.759461:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787829414.759462:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787829414.759463:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787829414.759464:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829414.759465:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829414.759466:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787829414.759468:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2d2c0 x1874670646827648/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829439 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787829414.759472:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829414.759473:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787829414.759475:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829414.759476:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829414.759477:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829414.759478:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787829414.759479:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787829414.759480:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829414.759482:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787829414.759483:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787829414.759485:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2d2c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646827648:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787829414.759487:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787829414.759488:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787829414.759489:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829414.759490:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787829414.759491:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829414.759492:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829414.759494:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646827776:192.168.203.143@tcp:8:kworker.0 00000100:00000001:1.0:1787829414.759495:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787829414.759496:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787829414.759497:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829414.759498:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787829414.759500:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829414.759503:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787829414.759503:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417871168 : -126234291680448 : ffff8d30c9e2cb40) 00000100:00000001:1.0:1787829414.759505:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1787829414.759510:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787829414.759512:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787829414.759514:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829414.759515:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829414.759516:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787829414.759517:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829414.759519:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298887715 00000100:00000001:1.0:1787829414.759521:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298888960 00000100:00000001:1.0:1787829414.759522:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298887715 00000100:00000001:1.0:1787829414.759524:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298888960 00000100:00000001:1.0:1787829414.759525:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829414.759527:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829414.759528:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787829414.759530:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829414/real 1787829414] req@ffff8d30c9e2cb40 x1874670646827776/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829439 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787829414.759534:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787829414.759535:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787829414.759536:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787829414.759537:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787829414.759538:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829414.759539:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829414.759540:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787829414.759542:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2cb40 x1874670646827776/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829439 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787829414.759546:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829414.759547:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787829414.759549:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829414.759550:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829414.759551:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829414.759552:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787829414.759553:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787829414.759554:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829414.759556:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787829414.759557:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787829414.759559:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646827776:192.168.203.143@tcp:8:kworker.0 00000100:00000001:1.0:1787829414.759561:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829414.759562:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829414.759563:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787829414.759564:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787829414.759566:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787829414.759567:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787829414.759568:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787829414.759569:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787829414.759570:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787829414.759572:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787829414.759573:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829414.759574:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829414.759575:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787829414.759575:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787829414.759577:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787829414.759577:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787829414.759578:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787829414.759579:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787829414.759580:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787829414.759581:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787829414.759582:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829414.759583:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829414.759583:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787829414.759584:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787829414.759585:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787829414.759585:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787829414.759586:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787829414.759587:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787829414.759588:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787829414.759588:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787829414.759589:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829414.759590:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829414.759592:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787829414.759593:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829414.759594:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787829414.759595:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829415.654683:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829415.654688:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829415.654748:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829415.654753:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829415.654756:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829416.678721:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829416.678726:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829416.678793:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829416.678797:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829416.678799:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829417.702679:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829417.702686:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829417.702753:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829417.702757:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829417.702760:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829418.726693:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829418.726699:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829418.726758:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829418.726761:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829418.726764:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829419.750711:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829419.750717:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829419.750751:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829419.750757:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829419.750761:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787829419.878697:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829419.878704:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1787829419.878706:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787829419.878710:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787829419.878713:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787829419.878718:0:468:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 4220 00000100:00080000:3.0:1787829419.878727:0:468:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787829419.878729:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829419.878730:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:3.0:1787829419.878734:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1787829419.878735:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1787829419.878738:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:3.0:1787829419.878739:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:3.0:1787829419.878741:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829419.878742:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:3.0:1787829419.878745:0:468:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:3.0:1787829419.878747:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829419.878749:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787829419.878751:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829419.878753:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829419.878754:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:3.0:1787829419.878756:0:468:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829419.878764:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787829419.878765:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829419.878767:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787829419.878770:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787829419.878773:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829419.878782:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829419.878789:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c7015a40 x1874670646827904/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787829419.878796:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787829419.878799:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829419.878812:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787829419.878814:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829419.878817:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1787829419.878818:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787829419.878819:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787829419.878821:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787829419.878823:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4220 00000100:00080000:3.0:1787829419.878825:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787829419.878826:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829419.878827:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:3.0:1787829419.878829:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1787829419.878830:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1787829419.878831:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000020:00000001:3.0:1787829419.878833:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:3.0:1787829419.878835:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829419.878836:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:3.0:1787829419.878838:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:3.0:1787829419.878840:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829419.878841:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787829419.878846:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829419.878847:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:3.0:1787829419.878848:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829419.878851:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:1.0:1787829419.878853:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000008:00100000:3.0:1787829419.878855:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00000001:1.0:1787829419.878857:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000008:00000001:3.0:1787829419.878859:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829419.878861:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787829419.878864:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829419.878864:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829419.878870:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787829419.878873:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1787829419.878874:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829419.878876:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829419.878877:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829419.878882:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00100000:1.0:1787829419.878885:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c7015a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646827904:192.168.203.143@tcp:250:kworker.0 02000000:00000001:3.0:1787829419.878886:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:1.0:1787829419.878892:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1787829419.878892:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829419.878897:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787829419.878899:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829419.878900:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829419.878901:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787829419.878907:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829419.878911:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c7016580 x1874670646828032/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787829419.878922:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787829419.878923:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787829419.878925:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829419.878927:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369542208 : -126234340009408 : ffff8d30c7015a40) 00000100:00000001:3.0:1787829419.878932:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829419.878934:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00080000:3.0:1787829419.878937:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829419.878942:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1787829419.878944:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787829419.878946:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787829419.878950:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787829419.878953:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4220 00000100:00080000:3.0:1787829419.878958:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787829419.878960:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829419.878962:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:3.0:1787829419.878965:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1787829419.878966:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1787829419.878968:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000020:00000001:3.0:1787829419.878971:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:3.0:1787829419.878973:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829419.878975:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:3.0:1787829419.878979:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:3.0:1787829419.878983:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:1.0:1787829419.878985:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 02000000:00000001:3.0:1787829419.878986:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787829419.878991:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829419.878994:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829419.878997:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:1.0:1787829419.879000:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000008:00100000:3.0:1787829419.879000:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1787829419.879006:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829419.879007:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829419.879009:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829419.879012:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829419.879015:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:1.0:1787829419.879016:0:2390:0:(events.c:68:request_out_callback()) Process leaving 02000000:00000001:3.0:1787829419.879019:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829419.879024:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:1.0:1787829419.879026:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829419.879026:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787829419.879031:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829419.879033:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4298894080 00000100:00000001:3.0:1787829419.879040:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829419.879043:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 00000100:00100000:3.0:1787829419.879049:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c70161c0 x1874670646828160/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787829419.879051:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4298894080 00000100:00000001:1.0:1787829419.879057:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 00000100:00000001:1.0:1787829419.879061:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829419.879067:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829419.879068:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787829419.879069:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00000001:3.0:1787829419.879073:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829419.879076:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829419/real 1787829419] req@ffff8d30c7015a40 x1874670646827904/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829444 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829419.879079:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787829419.879085:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000001:1.0:1787829419.879091:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00080000:3.0:1787829419.879092:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000400:00000001:1.0:1787829419.879095:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829419.879096:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787829419.879099:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787829419.879100:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829419.879102:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:1.0:1787829419.879104:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787829419.879107:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:3.0:1787829419.879108:0:468:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4220 00000100:00000001:1.0:1787829419.879111:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829419.879114:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:3.0:1787829419.879117:0:468:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1787829419.879119:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c7015a40 x1874670646827904/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829444 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829419.879122:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829419.879126:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:1.0:1787829419.879130:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:3.0:1787829419.879132:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:1.0:1787829419.879134:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000020:00000001:3.0:1787829419.879136:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:1.0:1787829419.879138:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829419.879140:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000020:00000001:3.0:1787829419.879141:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:1.0:1787829419.879142:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829419.879145:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000020:00000001:3.0:1787829419.879149:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:1.0:1787829419.879150:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787829419.879154:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00000001:3.0:1787829419.879156:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00080000:1.0:1787829419.879158:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829419.879160:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:1.0:1787829419.879162:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00080000:3.0:1787829419.879169:0:468:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:3.0:1787829419.879175:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829419.879191:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00100000:1.0:1787829419.879193:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c7015a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646827904:192.168.203.143@tcp:250:kworker.0 02000000:00000001:3.0:1787829419.879195:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829419.879199:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:3.0:1787829419.879199:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829419.879201:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:1.0:1787829419.879203:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000008:00100000:3.0:1787829419.879204:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00000001:1.0:1787829419.879206:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 00000008:00000001:3.0:1787829419.879209:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829419.879210:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 00000100:00000001:3.0:1787829419.879213:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829419.879215:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787829419.879218:0:2390:0:(genops.c:1217:class_import_put()) Process entered 02000000:00000001:3.0:1787829419.879218:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000020:00000001:1.0:1787829419.879220:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829419.879221:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829419.879224:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 00000100:00000001:3.0:1787829419.879224:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787829419.879226:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1787829419.879227:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 02000000:00000001:3.0:1787829419.879229:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829419.879231:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787829419.879233:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829419.879235:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829419.879236:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829419.879244:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c7014f00 x1874670646828288/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787829419.879246:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787829419.879249:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829419.879254:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1787829419.879255:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787829419.879257:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787829419.879259:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829419.879260:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787829419.879264:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787829419.879266:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829419.879266:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829419.879272:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787829419.879274:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829419.879276:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829419.879280:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c7016580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646828032:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787829419.879282:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787829419.879284:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787829419.879285:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829419.879287:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787829419.879289:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829419.879295:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787829419.879296:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369545088 : -126234340006528 : ffff8d30c7016580) 00000100:00000001:1.0:1787829419.879299:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1787829419.879309:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787829419.879312:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787829419.879314:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829419.879315:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829419.879316:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787829419.879318:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829419.879322:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298892822 00000100:00000001:1.0:1787829419.879324:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298894080 00000100:00000001:1.0:1787829419.879325:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298892822 00000100:00000001:1.0:1787829419.879327:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298894080 00000100:00000001:1.0:1787829419.879328:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829419.879330:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829419.879331:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787829419.879335:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829419/real 1787829419] req@ffff8d30c7016580 x1874670646828032/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829444 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787829419.879350:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787829419.879352:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787829419.879355:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787829419.879356:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787829419.879358:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829419.879360:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829419.879361:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787829419.879366:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c7016580 x1874670646828032/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829444 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787829419.879374:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829419.879376:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787829419.879379:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829419.879380:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829419.879382:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829419.879384:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787829419.879385:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787829419.879388:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829419.879391:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787829419.879393:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787829419.879395:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c7016580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646828032:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787829419.879398:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787829419.879399:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787829419.879400:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829419.879402:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787829419.879403:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829419.879404:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829419.879406:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c70161c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646828160:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787829419.879408:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787829419.879410:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787829419.879411:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829419.879412:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787829419.879415:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829419.879418:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787829419.879419:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369544128 : -126234340007488 : ffff8d30c70161c0) 00000100:00000001:1.0:1787829419.879422:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1787829419.879431:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787829419.879434:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787829419.879436:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829419.879437:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829419.879438:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787829419.879440:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829419.879442:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298892822 00000100:00000001:1.0:1787829419.879444:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298894080 00000100:00000001:1.0:1787829419.879445:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298892822 00000100:00000001:1.0:1787829419.879447:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298894080 00000100:00000001:1.0:1787829419.879448:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829419.879449:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829419.879450:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787829419.879453:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829419/real 1787829419] req@ffff8d30c70161c0 x1874670646828160/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829444 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787829419.879458:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787829419.879458:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787829419.879460:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787829419.879461:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787829419.879462:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829419.879463:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829419.879464:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787829419.879466:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c70161c0 x1874670646828160/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829444 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787829419.879470:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829419.879471:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787829419.879473:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829419.879474:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829419.879475:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829419.879476:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787829419.879477:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787829419.879479:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829419.879481:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787829419.879482:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787829419.879484:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c70161c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646828160:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787829419.879486:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829419.879487:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829419.879488:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787829419.879489:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787829419.879490:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787829419.879491:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787829419.879492:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787829419.879494:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787829419.879494:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787829419.879496:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787829419.879496:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829419.879497:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829419.879498:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787829419.879499:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787829419.879501:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787829419.879501:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787829419.879502:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787829419.879503:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787829419.879504:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787829419.879505:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787829419.879506:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829419.879507:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829419.879509:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787829419.879510:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829419.879512:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787829419.879513:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1787829419.879513:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787829419.879514:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787829419.879516:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829419.879517:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787829419.879518:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829419.879519:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829419.879521:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c7014f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646828288:192.168.203.143@tcp:8:kworker.0 00000100:00000001:1.0:1787829419.879523:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787829419.879524:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787829419.879525:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829419.879526:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787829419.879527:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829419.879531:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787829419.879531:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369539328 : -126234340012288 : ffff8d30c7014f00) 00000100:00000001:1.0:1787829419.879534:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1787829419.879540:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787829419.879542:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787829419.879543:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829419.879545:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829419.879546:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787829419.879547:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829419.879550:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298892835 00000100:00000001:1.0:1787829419.879551:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298894080 00000100:00000001:1.0:1787829419.879553:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298892835 00000100:00000001:1.0:1787829419.879554:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298894080 00000100:00000001:1.0:1787829419.879556:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829419.879557:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829419.879558:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787829419.879560:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829419/real 1787829419] req@ffff8d30c7014f00 x1874670646828288/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829444 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787829419.879565:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787829419.879566:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787829419.879567:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787829419.879568:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787829419.879569:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829419.879570:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829419.879571:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787829419.879574:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c7014f00 x1874670646828288/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829444 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787829419.879578:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829419.879579:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787829419.879581:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829419.879582:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829419.879582:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829419.879583:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787829419.879584:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787829419.879586:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829419.879588:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787829419.879589:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787829419.879591:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c7014f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646828288:192.168.203.143@tcp:8:kworker.0 00000100:00000001:1.0:1787829419.879593:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829419.879594:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829419.879594:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787829419.879595:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787829419.879596:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787829419.879597:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787829419.879598:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787829419.879599:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787829419.879621:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787829419.879623:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787829419.879624:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829419.879625:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829419.879627:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787829419.879628:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829419.879629:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787829419.879630:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829420.774683:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829420.774690:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829420.774731:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829420.774734:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829420.774737:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829421.798691:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829421.798696:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829421.798747:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829421.798751:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829421.798755:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829422.822707:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829422.822714:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829422.822776:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829422.822781:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829422.822784:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829423.846684:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829423.846689:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829423.846742:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829423.846745:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829423.846747:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829424.870718:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829424.870726:0:815:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1787829424.870728:0:815:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1787829424.870731:0:815:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829424.870736:0:815:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1787829424.870739:0:815:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829424.870741:0:815:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829424.870743:0:815:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829424.870808:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829424.870812:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829424.870815:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787829424.998766:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829424.998776:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1787829424.998779:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787829424.998782:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787829424.998787:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787829424.998795:0:40798:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 4225 00000100:00080000:0.0:1787829424.998807:0:40798:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787829424.998810:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829424.998812:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787829424.998816:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787829424.998819:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787829424.998822:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:0.0:1787829424.998823:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:0.0:1787829424.998825:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829424.998826:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:0.0:1787829424.998830:0:40798:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787829424.998832:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829424.998835:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787829424.998839:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787829424.998841:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829424.998843:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1787829424.998845:0:40798:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829424.998859:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787829424.998861:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829424.998865:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787829424.998868:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787829424.998874:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829424.998888:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787829424.998900:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d270c0 x1874670646828416/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787829424.998912:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787829424.998917:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829424.998935:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787829424.998938:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829424.998940:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1787829424.998942:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787829424.998944:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787829424.998946:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787829424.998948:0:40798:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4225 00000100:00080000:0.0:1787829424.998952:0:40798:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787829424.998953:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829424.998955:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787829424.998958:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787829424.998959:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787829424.998961:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000020:00000001:0.0:1787829424.998963:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:0.0:1787829424.998965:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829424.998967:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:0.0:1787829424.998971:0:40798:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787829424.998973:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829424.998975:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787829424.998980:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787829424.998982:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829424.998983:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:1.0:1787829424.998985:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000008:00100000:0.0:1787829424.998988:0:40798:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1787829424.998994:0:40798:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829424.998997:0:40798:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829424.998998:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 02000000:00000001:0.0:1787829424.999002:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:1.0:1787829424.999004:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 02000000:00000001:0.0:1787829424.999005:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829424.999008:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787829424.999010:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:1.0:1787829424.999010:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 02000000:00000001:0.0:1787829424.999014:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829424.999016:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829424.999020:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787829424.999026:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d25a40 x1874670646828544/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:1.0:1787829424.999027:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787829424.999034:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787829424.999038:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787829424.999043:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829424.999044:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829424.999048:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787829424.999051:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829424.999055:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1787829424.999056:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787829424.999058:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1787829424.999059:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d270c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646828416:192.168.203.143@tcp:250:kworker.0 00000100:00000001:0.0:1787829424.999060:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787829424.999065:0:40798:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4225 00000100:00080000:0.0:1787829424.999070:0:40798:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787829424.999075:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787829424.999078:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:0.0:1787829424.999079:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787829424.999087:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787829424.999088:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787829424.999090:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000020:00000001:0.0:1787829424.999091:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 02000000:00000001:1.0:1787829424.999092:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:0.0:1787829424.999093:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829424.999097:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 02000000:00000001:1.0:1787829424.999100:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787829424.999103:0:40798:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787829424.999108:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829424.999109:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1787829424.999110:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787829424.999115:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787829424.999117:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829424.999118:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1787829424.999120:0:40798:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1787829424.999121:0:40798:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829424.999122:0:40798:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829424.999123:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829424.999126:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787829424.999128:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829424.999129:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787829424.999130:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787829424.999131:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829424.999134:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787829424.999154:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d24f00 x1874670646828672/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787829424.999165:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787829424.999167:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829424.999171:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829424.999172:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00080000:0.0:1787829424.999174:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1787829424.999177:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601348800 : -126234108202816 : ffff8d30d4d270c0) 00000100:00080000:0.0:1787829424.999179:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1787829424.999181:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787829424.999183:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787829424.999189:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787829424.999192:0:40798:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4225 00000100:00000001:1.0:1787829424.999192:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00080000:0.0:1787829424.999200:0:40798:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787829424.999202:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829424.999203:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787829424.999206:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787829424.999207:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787829424.999209:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000020:00000001:0.0:1787829424.999210:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:0.0:1787829424.999212:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829424.999214:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:0.0:1787829424.999217:0:40798:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787829424.999219:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829424.999220:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787829424.999223:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787829424.999224:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829424.999226:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1787829424.999228:0:40798:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1787829424.999230:0:40798:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829424.999231:0:40798:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829424.999235:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787829424.999236:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829424.999238:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787829424.999239:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787829424.999241:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829424.999245:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787829424.999248:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d24000 x1874670646828800/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787829424.999254:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787829424.999255:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829424.999257:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829424.999262:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829424.999271:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4298899200 00000100:00000001:1.0:1787829424.999285:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 00000100:00000001:1.0:1787829424.999291:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4298899200 00000100:00000001:1.0:1787829424.999296:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 00000100:00000001:1.0:1787829424.999300:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829424.999303:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829424.999306:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829424.999309:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829424.999317:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787829424.999319:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:1.0:1787829424.999323:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787829424.999333:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1787829424.999334:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787829424.999336:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787829424.999338:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829424.999341:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787829424.999343:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829424.999346:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829424.999352:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d25a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646828544:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787829424.999356:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787829424.999359:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787829424.999361:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829424.999363:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787829424.999367:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829424.999375:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787829424.999377:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601343040 : -126234108208576 : ffff8d30d4d25a40) 00000100:00000001:1.0:1787829424.999383:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:1.0:1787829424.999399:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829424.999405:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298897942 00000100:00000001:1.0:1787829424.999407:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298899200 00000100:00000001:1.0:1787829424.999410:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298897942 00000100:00000001:1.0:1787829424.999412:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298899200 00000100:00000001:1.0:1787829424.999415:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829424.999417:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829424.999419:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787829424.999421:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787829424.999423:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829424.999426:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787829424.999427:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829424.999430:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829424.999435:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d24f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646828672:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787829424.999439:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787829424.999441:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787829424.999444:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829424.999446:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787829424.999450:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829424.999456:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787829424.999458:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601340160 : -126234108211456 : ffff8d30d4d24f00) 00000100:00000001:1.0:1787829424.999462:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:1.0:1787829424.999472:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829424.999476:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298897942 00000100:00000001:1.0:1787829424.999479:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298899200 00000100:00000001:1.0:1787829424.999481:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298897942 00000100:00000001:1.0:1787829424.999483:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298899200 00000100:00000001:1.0:1787829424.999485:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829424.999487:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829424.999489:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787829424.999491:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787829424.999492:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829424.999496:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787829424.999497:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829424.999500:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829424.999503:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d24000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646828800:192.168.203.143@tcp:8:kworker.0 00000100:00000001:1.0:1787829424.999507:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787829424.999510:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787829424.999511:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829424.999513:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787829424.999516:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829424.999522:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787829424.999524:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601336320 : -126234108215296 : ffff8d30d4d24000) 00000100:00000001:1.0:1787829424.999527:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:1.0:1787829424.999537:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829424.999541:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298897955 00000100:00000001:1.0:1787829424.999543:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298899200 00000100:00000001:1.0:1787829424.999545:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298897955 00000100:00000001:1.0:1787829424.999548:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298899200 00000100:00000001:1.0:1787829424.999550:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829424.999552:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829424.999555:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829424.999557:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829424.999560:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787829424.999562:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:1.0:1787829424.999564:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787829424.999565:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1787829424.999567:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829424.999569:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829425.894687:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829425.894695:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829425.894746:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829425.894750:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829425.894753:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829426.918723:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829426.918729:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829426.918814:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829426.918819:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829426.918822:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829427.942702:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829427.942707:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829427.942761:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829427.942763:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829427.942766:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:1.0:1787829428.070869:0:2380:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.143:988 00000400:00000100:1.0:1787829428.070917:0:2380:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.143@tcp at host 192.168.203.143 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1787829428.070934:0:2380:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.143@tcp 00000100:00000001:1.0:1787829428.070945:0:2380:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787829428.070962:0:2380:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829428.070965:0:2380:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829428.070967:0:2380:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:1.0:1787829428.070977:0:2380:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.143@tcp 00000100:00000001:1.0:1787829428.070984:0:2380:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787829428.070987:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787829428.070988:0:2380:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829428.070991:0:2380:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829428.070995:0:2380:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1787829428.070995:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787829428.071001:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000800:00000100:1.0:1787829428.071006:0:2380:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.143@tcp 00000100:00000001:1.0:1787829428.071013:0:2380:0:(events.c:34:request_out_callback()) Process entered 00000100:00100000:3.0:1787829428.071015:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829424/real 1787829428] req@ffff8d30d4d25a40 x1874670646828544/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829449 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787829428.071017:0:2380:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829428.071020:0:2380:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829428.071024:0:2380:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:1.0:1787829428.071034:0:2380:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.143@tcp 00000400:00000001:3.0:1787829428.071037:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787829428.071041:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829428.071048:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787829428.071050:0:2380:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787829428.071052:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787829428.071057:0:2380:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829428.071061:0:2380:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829428.071063:0:2380:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1787829428.071063:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829428.071069:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829428.071072:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829428.071079:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d25a40 x1874670646828544/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829449 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829428.071089:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829428.071094:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829428.071098:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829428.071099:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829428.071101:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829428.071104:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829428.071107:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829428.071111:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829428.071116:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829428.071119:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829428.071137:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d25a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646828544:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829428.071143:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1787829428.071148:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829424/real 1787829428] req@ffff8d30d4d24f00 x1874670646828672/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829449 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1787829428.071158:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787829428.071159:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829428.071163:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829428.071165:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829428.071166:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829428.071168:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829428.071170:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829428.071174:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d24f00 x1874670646828672/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829449 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829428.071183:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829428.071185:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829428.071188:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829428.071190:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829428.071192:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829428.071194:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829428.071196:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829428.071199:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829428.071203:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829428.071205:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829428.071209:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d24f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646828672:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829428.071214:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1787829428.071218:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829424/real 1787829428] req@ffff8d30d4d24000 x1874670646828800/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829449 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1787829428.071227:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787829428.071228:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829428.071232:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829428.071234:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829428.071235:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829428.071238:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829428.071239:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829428.071244:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d24000 x1874670646828800/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829449 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829428.071252:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829428.071254:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829428.071257:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829428.071258:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829428.071259:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829428.071261:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829428.071264:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829428.071266:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829428.071271:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829428.071273:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829428.071276:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d24000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646828800:192.168.203.143@tcp:8:kworker.0 00000100:00000001:3.0:1787829428.071282:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1787829428.071286:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829424/real 1787829428] req@ffff8d30d4d270c0 x1874670646828416/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829449 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1787829428.071295:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787829428.071296:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829428.071299:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829428.071301:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829428.071303:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829428.071305:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829428.071306:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829428.071311:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d270c0 x1874670646828416/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829449 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829428.071318:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829428.071320:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829428.071323:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829428.071325:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829428.071326:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829428.071328:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829428.071330:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829428.071332:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829428.071336:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829428.071338:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829428.071341:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d270c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646828416:192.168.203.143@tcp:250:kworker.0 00000100:00000001:3.0:1787829428.071345:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829428.071348:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829428.071349:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829428.071353:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829428.071358:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829428.071361:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829428.071363:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829428.071366:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829428.071369:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829428.071373:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829428.071374:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829428.071376:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829428.071377:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829428.071379:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829428.071381:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829428.071383:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829428.071384:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829428.071386:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829428.071387:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829428.071389:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829428.071391:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829428.071392:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829428.071393:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829428.071395:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829428.071397:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829428.071398:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829428.071399:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829428.071401:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829428.071402:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829428.071404:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829428.071405:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829428.071406:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829428.071408:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829428.071409:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829428.071412:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829428.071413:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829428.071414:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829428.071417:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829428.071418:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829428.071421:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829428.071423:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829428.071425:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829428.071433:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787829428.071436:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829428.071438:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1787829428.071440:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829428.966722:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829428.966727:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829428.966787:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829428.966791:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829428.966795:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829429.990717:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829429.990723:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829429.990792:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829429.990795:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829429.990798:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787829430.118774:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829430.118782:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1787829430.118785:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787829430.118788:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787829430.118792:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787829430.118798:0:40798:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 4230 00000100:00080000:3.0:1787829430.118807:0:40798:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787829430.118810:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829430.118811:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:3.0:1787829430.118814:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1787829430.118816:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1787829430.118819:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:3.0:1787829430.118822:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:3.0:1787829430.118824:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829430.118825:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:3.0:1787829430.118831:0:40798:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:3.0:1787829430.118834:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829430.118836:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787829430.118840:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829430.118844:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829430.118846:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:3.0:1787829430.118849:0:40798:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829430.118858:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787829430.118860:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829430.118863:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787829430.118866:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787829430.118870:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829430.118882:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829430.118892:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c70143c0 x1874670646828928/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787829430.118904:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787829430.118910:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829430.118927:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787829430.118929:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829430.118933:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1787829430.118935:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787829430.118936:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787829430.118940:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787829430.118943:0:40798:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4230 00000100:00080000:3.0:1787829430.118948:0:40798:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787829430.118951:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829430.118953:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:3.0:1787829430.118957:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1787829430.118959:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:1.0:1787829430.118962:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:3.0:1787829430.118962:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000020:00000001:3.0:1787829430.118968:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:1.0:1787829430.118969:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787829430.118971:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787829430.118974:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787829430.118975:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:1.0:1787829430.118979:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00080000:3.0:1787829430.118983:0:40798:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:1.0:1787829430.118984:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829430.118989:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829430.118993:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787829430.118994:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1787829430.118998:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829430.119003:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829430.119004:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829430.119010:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829430.119011:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c70143c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646828928:192.168.203.143@tcp:250:kworker.0 00000100:00000001:3.0:1787829430.119014:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:1.0:1787829430.119020:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000008:00100000:3.0:1787829430.119022:0:40798:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 02000000:00000001:1.0:1787829430.119026:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000008:00000001:3.0:1787829430.119028:0:40798:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829430.119031:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829430.119032:0:40798:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829430.119037:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1787829430.119041:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1787829430.119044:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829430.119045:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829430.119050:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787829430.119052:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787829430.119053:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829430.119061:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829430.119066:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00100000:3.0:1787829430.119069:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c7015680 x1874670646829056/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787829430.119071:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369536448 : -126234340015168 : ffff8d30c70143c0) 00000100:00000001:1.0:1787829430.119078:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:3.0:1787829430.119089:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787829430.119092:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829430.119095:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787829430.119099:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829430.119102:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1787829430.119104:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787829430.119120:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787829430.119124:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787829430.119127:0:40798:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4230 00000100:00080000:3.0:1787829430.119132:0:40798:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787829430.119135:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829430.119138:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000800:00000100:1.0:1787829430.119140:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000020:00000001:3.0:1787829430.119144:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1787829430.119148:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:1.0:1787829430.119157:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000020:00000001:3.0:1787829430.119157:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:1.0:1787829430.119164:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000020:00000001:3.0:1787829430.119167:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:1.0:1787829430.119169:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829430.119173:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787829430.119174:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1787829430.119178:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:1.0:1787829430.119184:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787829430.119186:0:40798:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:1.0:1787829430.119192:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4298904320 00000100:00000001:3.0:1787829430.119192:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829430.119198:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787829430.119203:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829430.119206:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829430.119208:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 00000100:00000001:3.0:1787829430.119209:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:3.0:1787829430.119213:0:40798:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00000001:1.0:1787829430.119218:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4298904320 00000008:00000001:3.0:1787829430.119218:0:40798:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829430.119224:0:40798:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829430.119228:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 02000000:00000001:3.0:1787829430.119231:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:1.0:1787829430.119235:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829430.119235:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829430.119242:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:1.0:1787829430.119243:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829430.119248:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 02000000:00000001:3.0:1787829430.119251:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787829430.119255:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829430.119256:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829430/real 1787829430] req@ffff8d30c70143c0 x1874670646828928/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829455 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829430.119265:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:1.0:1787829430.119276:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00100000:3.0:1787829430.119276:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c70152c0 x1874670646829184/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000001:1.0:1787829430.119280:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787829430.119286:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787829430.119291:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829430.119291:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787829430.119296:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829430.119296:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829430.119301:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829430.119304:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829430.119306:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:3.0:1787829430.119308:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829430.119315:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:1.0:1787829430.119316:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c70143c0 x1874670646828928/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829455 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829430.119318:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787829430.119322:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787829430.119325:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787829430.119330:0:40798:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4230 00000100:00080000:3.0:1787829430.119336:0:40798:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1787829430.119342:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829430.119342:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829430.119345:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:1.0:1787829430.119350:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000020:00000001:3.0:1787829430.119350:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1787829430.119352:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1787829430.119357:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:1.0:1787829430.119358:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829430.119363:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829430.119365:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829430.119367:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000020:00000001:3.0:1787829430.119369:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:1.0:1787829430.119371:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829430.119375:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00080000:1.0:1787829430.119378:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00000001:3.0:1787829430.119379:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:3.0:1787829430.119386:0:40798:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00080000:1.0:1787829430.119389:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829430.119391:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829430.119396:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 02000000:00000001:3.0:1787829430.119396:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787829430.119401:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00100000:1.0:1787829430.119405:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c70143c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646828928:192.168.203.143@tcp:250:kworker.0 02000000:00000001:3.0:1787829430.119407:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829430.119413:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:1.0:1787829430.119416:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829430.119421:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000008:00100000:3.0:1787829430.119422:0:40798:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00000001:1.0:1787829430.119423:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787829430.119427:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 00000008:00000001:3.0:1787829430.119429:0:40798:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829430.119433:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00000001:3.0:1787829430.119433:0:40798:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829430.119438:0:2390:0:(genops.c:1217:class_import_put()) Process entered 02000000:00000001:3.0:1787829430.119441:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000020:00000001:1.0:1787829430.119442:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829430.119445:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829430.119448:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 00000100:00000001:3.0:1787829430.119450:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1787829430.119453:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 02000000:00000001:3.0:1787829430.119454:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:1.0:1787829430.119457:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 02000000:00000001:3.0:1787829430.119458:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829430.119461:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829430.119466:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829430.119467:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829430.119475:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00100000:3.0:1787829430.119477:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c7014b40 x1874670646829312/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787829430.119479:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829430.119484:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787829430.119487:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1787829430.119489:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787829430.119492:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787829430.119494:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787829430.119496:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829430.119501:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787829430.119503:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829430.119515:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829430.119520:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c7015680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646829056:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787829430.119524:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787829430.119526:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787829430.119528:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829430.119529:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787829430.119532:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829430.119539:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787829430.119540:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369541248 : -126234340010368 : ffff8d30c7015680) 00000100:00000001:1.0:1787829430.119545:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1787829430.119559:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787829430.119565:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787829430.119568:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829430.119570:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829430.119572:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787829430.119576:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829430.119582:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298903062 00000100:00000001:1.0:1787829430.119586:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298904320 00000100:00000001:1.0:1787829430.119588:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298903062 00000100:00000001:1.0:1787829430.119591:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298904320 00000100:00000001:1.0:1787829430.119594:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829430.119596:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829430.119598:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787829430.119633:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829430/real 1787829430] req@ffff8d30c7015680 x1874670646829056/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829455 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829430.119637:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829430.119645:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:1.0:1787829430.119659:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787829430.119661:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787829430.119666:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787829430.119669:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787829430.119671:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829430.119674:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829430.119676:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787829430.119689:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c7015680 x1874670646829056/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829455 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787829430.119697:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829430.119700:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787829430.119703:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829430.119705:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829430.119706:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829430.119708:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787829430.119710:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787829430.119713:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829430.119716:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787829430.119719:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787829430.119723:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c7015680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646829056:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787829430.119727:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787829430.119729:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787829430.119731:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829430.119733:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787829430.119735:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829430.119738:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829430.119755:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c70152c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646829184:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787829430.119760:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787829430.119762:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787829430.119765:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829430.119766:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787829430.119769:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829430.119774:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787829430.119776:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369540288 : -126234340011328 : ffff8d30c70152c0) 00000100:00000001:1.0:1787829430.119780:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1787829430.119794:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787829430.119799:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787829430.119802:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829430.119803:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829430.119805:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787829430.119807:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829430.119811:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298903063 00000100:00000001:1.0:1787829430.119813:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298904320 00000100:00000001:1.0:1787829430.119816:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298903063 00000100:00000001:1.0:1787829430.119818:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298904320 00000100:00000001:1.0:1787829430.119819:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829430.119821:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829430.119823:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787829430.119827:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829430/real 1787829430] req@ffff8d30c70152c0 x1874670646829184/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829455 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787829430.119836:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787829430.119838:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787829430.119840:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787829430.119842:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787829430.119843:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829430.119845:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829430.119846:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787829430.119850:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c70152c0 x1874670646829184/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829455 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787829430.119856:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829430.119858:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787829430.119861:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829430.119862:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829430.119863:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829430.119865:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787829430.119866:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787829430.119869:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829430.119872:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787829430.119874:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787829430.119877:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c70152c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646829184:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787829430.119881:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829430.119883:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829430.119884:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787829430.119885:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787829430.119888:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787829430.119889:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787829430.119891:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787829430.119893:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787829430.119894:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787829430.119896:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787829430.119898:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829430.119900:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829430.119901:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787829430.119902:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787829430.119904:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787829430.119905:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787829430.119906:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787829430.119908:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787829430.119909:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787829430.119910:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787829430.119911:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829430.119913:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829430.119917:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787829430.119918:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829430.119920:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787829430.119922:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1787829430.119923:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787829430.119925:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787829430.119927:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829430.119929:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787829430.119931:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829430.119933:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829430.119937:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c7014b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646829312:192.168.203.143@tcp:8:kworker.0 00000100:00000001:1.0:1787829430.119941:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787829430.119943:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787829430.119944:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829430.119946:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787829430.119949:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829430.119955:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787829430.119956:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369538368 : -126234340013248 : ffff8d30c7014b40) 00000100:00000001:1.0:1787829430.119959:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1787829430.119970:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787829430.119975:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787829430.119977:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829430.119979:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829430.119980:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787829430.119983:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829430.119987:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298903076 00000100:00000001:1.0:1787829430.119989:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298904320 00000100:00000001:1.0:1787829430.119991:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298903076 00000100:00000001:1.0:1787829430.119993:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298904320 00000100:00000001:1.0:1787829430.119995:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829430.119997:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829430.119999:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787829430.120003:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829430/real 1787829430] req@ffff8d30c7014b40 x1874670646829312/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829455 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787829430.120011:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787829430.120012:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787829430.120015:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787829430.120017:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787829430.120018:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829430.120020:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829430.120022:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787829430.120025:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c7014b40 x1874670646829312/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829455 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787829430.120032:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829430.120034:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787829430.120037:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829430.120039:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829430.120041:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829430.120042:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787829430.120044:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787829430.120047:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829430.120050:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787829430.120052:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787829430.120056:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c7014b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646829312:192.168.203.143@tcp:8:kworker.0 00000100:00000001:1.0:1787829430.120059:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829430.120061:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829430.120062:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787829430.120063:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787829430.120065:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787829430.120067:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787829430.120068:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787829430.120070:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787829430.120071:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787829430.120073:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787829430.120074:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829430.120076:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829430.120079:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787829430.120080:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829430.120082:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787829430.120083:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829431.014676:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829431.014681:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829431.014739:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829431.014743:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829431.014747:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829432.038689:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829432.038696:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829432.038751:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829432.038755:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829432.038759:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829433.062690:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829433.062697:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829433.062751:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829433.062755:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829433.062759:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829434.086698:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829434.086705:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829434.086756:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829434.086759:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829434.086762:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829435.110679:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829435.110686:0:815:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1787829435.110687:0:815:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1787829435.110690:0:815:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829435.110694:0:815:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1787829435.110696:0:815:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829435.110697:0:815:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829435.110699:0:815:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829435.110752:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829435.110756:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829435.110760:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787829435.238706:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829435.238714:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1787829435.238717:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787829435.238720:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787829435.238725:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787829435.238731:0:40798:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 4235 00000100:00080000:0.0:1787829435.238741:0:40798:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787829435.238743:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829435.238745:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787829435.238749:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787829435.238751:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787829435.238754:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:0.0:1787829435.238756:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:0.0:1787829435.238758:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829435.238760:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:0.0:1787829435.238763:0:40798:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787829435.238765:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829435.238768:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787829435.238770:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787829435.238773:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829435.238775:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1787829435.238777:0:40798:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829435.238784:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787829435.238787:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829435.238789:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787829435.238791:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787829435.238794:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829435.238804:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787829435.238818:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d270c0 x1874670646829440/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787829435.238826:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787829435.238829:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829435.238841:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787829435.238844:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829435.238847:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1787829435.238848:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787829435.238850:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787829435.238852:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787829435.238855:0:40798:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4235 00000100:00080000:0.0:1787829435.238859:0:40798:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787829435.238861:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829435.238863:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787829435.238865:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787829435.238867:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:3.0:1787829435.238867:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:0.0:1787829435.238872:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:3.0:1787829435.238874:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000020:00000001:0.0:1787829435.238875:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:0.0:1787829435.238878:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829435.238878:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1787829435.238883:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:3.0:1787829435.238883:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787829435.238887:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787829435.238892:0:40798:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 02000000:00000001:3.0:1787829435.238893:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787829435.238896:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787829435.238898:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829435.238899:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829435.238905:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00100000:3.0:1787829435.238907:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d270c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646829440:192.168.203.143@tcp:250:kworker.0 02000000:00000001:0.0:1787829435.238913:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829435.238913:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1787829435.238918:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787829435.238919:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829435.238921:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829435.238923:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1787829435.238930:0:40798:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1787829435.238934:0:40798:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829435.238936:0:40798:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829435.238941:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787829435.238942:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829435.238944:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787829435.238946:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787829435.238947:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829435.238953:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787829435.238960:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d24000 x1874670646829568/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787829435.238969:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787829435.238971:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829435.238975:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787829435.238978:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829435.238981:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1787829435.238983:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787829435.238985:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787829435.238987:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787829435.238990:0:40798:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4235 00000100:00080000:0.0:1787829435.238993:0:40798:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787829435.238995:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829435.238996:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787829435.238998:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787829435.238999:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787829435.239001:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000020:00000001:0.0:1787829435.239002:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:0.0:1787829435.239004:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829435.239005:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:0.0:1787829435.239008:0:40798:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787829435.239009:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829435.239011:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787829435.239013:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787829435.239015:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829435.239016:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1787829435.239018:0:40798:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1787829435.239020:0:40798:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829435.239021:0:40798:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829435.239024:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787829435.239025:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829435.239026:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787829435.239027:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787829435.239028:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829435.239032:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787829435.239035:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d24f00 x1874670646829696/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787829435.239041:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787829435.239043:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829435.239044:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787829435.239046:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829435.239049:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1787829435.239050:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787829435.239051:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787829435.239052:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787829435.239054:0:40798:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4235 00000100:00080000:0.0:1787829435.239056:0:40798:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787829435.239057:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829435.239058:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787829435.239059:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787829435.239060:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787829435.239061:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000020:00000001:0.0:1787829435.239062:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:0.0:1787829435.239063:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829435.239065:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:0.0:1787829435.239067:0:40798:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787829435.239069:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829435.239084:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787829435.239086:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787829435.239088:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829435.239090:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1787829435.239091:0:40798:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1787829435.239093:0:40798:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829435.239094:0:40798:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829435.239097:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787829435.239098:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829435.239099:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787829435.239100:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787829435.239102:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829435.239106:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787829435.239110:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d25a40 x1874670646829824/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787829435.239115:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787829435.239116:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829435.239118:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829435.239134:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1787829435.239140:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829435.239154:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787829435.239156:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601348800 : -126234108202816 : ffff8d30d4d270c0) 00000100:00000001:3.0:1787829435.239160:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1787829435.239228:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:3.0:1787829435.239237:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787829435.239241:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829435.239243:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829435.239244:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1787829435.239248:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829435.239251:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4298909440 00000100:00000001:3.0:1787829435.239258:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 00000100:00000001:3.0:1787829435.239262:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4298909440 00000100:00000001:3.0:1787829435.239265:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 00000100:00000001:3.0:1787829435.239267:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829435.239269:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829435.239271:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1787829435.239275:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829435/real 1787829435] req@ffff8d30d4d270c0 x1874670646829440/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829460 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1787829435.239283:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787829435.239284:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829435.239287:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829435.239289:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829435.239290:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829435.239291:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829435.239294:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829435.239298:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d270c0 x1874670646829440/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829460 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829435.239306:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829435.239310:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829435.239313:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829435.239314:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829435.239316:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829435.239318:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829435.239320:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829435.239323:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829435.239327:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829435.239330:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829435.239335:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d270c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646829440:192.168.203.143@tcp:250:kworker.0 00000100:00000001:3.0:1787829435.239341:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829435.239343:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829435.239345:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829435.239347:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829435.239350:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829435.239352:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829435.239354:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829435.239357:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829435.239358:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829435.239361:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829435.239362:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829435.239364:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829435.239369:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787829435.239371:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829435.239372:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1787829435.239374:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787829435.239375:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787829435.239376:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787829435.239377:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829435.239379:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787829435.239380:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829435.239382:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829435.239384:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d24000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646829568:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829435.239386:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1787829435.239388:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1787829435.239389:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829435.239390:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1787829435.239392:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829435.239397:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787829435.239398:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601336320 : -126234108215296 : ffff8d30d4d24000) 00000100:00000001:3.0:1787829435.239402:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1787829435.239411:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:3.0:1787829435.239415:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787829435.239417:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829435.239418:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829435.239419:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1787829435.239421:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829435.239424:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298908182 00000100:00000001:3.0:1787829435.239426:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298909440 00000100:00000001:3.0:1787829435.239428:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298908182 00000100:00000001:3.0:1787829435.239429:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298909440 00000100:00000001:3.0:1787829435.239431:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829435.239432:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829435.239433:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1787829435.239436:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829435/real 1787829435] req@ffff8d30d4d24000 x1874670646829568/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829460 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1787829435.239441:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787829435.239442:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829435.239444:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829435.239445:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829435.239446:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829435.239447:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829435.239448:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829435.239450:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d24000 x1874670646829568/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829460 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829435.239455:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829435.239457:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829435.239459:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829435.239460:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829435.239460:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829435.239461:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829435.239463:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829435.239464:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829435.239466:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829435.239467:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829435.239470:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d24000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646829568:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829435.239472:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787829435.239473:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787829435.239474:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829435.239476:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787829435.239477:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829435.239478:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829435.239480:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d24f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646829696:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829435.239482:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1787829435.239483:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1787829435.239484:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829435.239485:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1787829435.239487:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829435.239490:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787829435.239491:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601340160 : -126234108211456 : ffff8d30d4d24f00) 00000100:00000001:3.0:1787829435.239493:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1787829435.239499:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:3.0:1787829435.239502:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787829435.239503:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829435.239504:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829435.239505:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1787829435.239507:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829435.239509:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298908182 00000100:00000001:3.0:1787829435.239511:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298909440 00000100:00000001:3.0:1787829435.239512:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298908182 00000100:00000001:3.0:1787829435.239513:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298909440 00000100:00000001:3.0:1787829435.239515:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829435.239516:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829435.239517:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1787829435.239519:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829435/real 1787829435] req@ffff8d30d4d24f00 x1874670646829696/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829460 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1787829435.239529:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787829435.239529:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829435.239531:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829435.239532:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829435.239533:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829435.239534:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829435.239534:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829435.239537:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d24f00 x1874670646829696/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829460 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829435.239541:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829435.239542:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829435.239543:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829435.239544:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829435.239545:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829435.239546:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829435.239547:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829435.239548:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829435.239550:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829435.239552:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829435.239553:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d24f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646829696:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829435.239556:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787829435.239557:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787829435.239558:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829435.239559:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787829435.239560:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829435.239562:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829435.239563:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d25a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646829824:192.168.203.143@tcp:8:kworker.0 00000100:00000001:3.0:1787829435.239565:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1787829435.239566:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1787829435.239568:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829435.239569:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1787829435.239570:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829435.239573:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787829435.239574:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601343040 : -126234108208576 : ffff8d30d4d25a40) 00000100:00000001:3.0:1787829435.239577:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1787829435.239582:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:3.0:1787829435.239585:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787829435.239586:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829435.239587:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829435.239588:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1787829435.239589:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829435.239591:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298908195 00000100:00000001:3.0:1787829435.239592:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298909440 00000100:00000001:3.0:1787829435.239594:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298908195 00000100:00000001:3.0:1787829435.239595:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298909440 00000100:00000001:3.0:1787829435.239597:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829435.239598:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829435.239623:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1787829435.239626:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829435/real 1787829435] req@ffff8d30d4d25a40 x1874670646829824/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829460 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1787829435.239631:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787829435.239632:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829435.239636:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829435.239637:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829435.239639:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829435.239640:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829435.239642:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829435.239645:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d25a40 x1874670646829824/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829460 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829435.239650:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829435.239652:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829435.239655:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829435.239657:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829435.239658:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829435.239659:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829435.239661:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829435.239663:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829435.239666:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829435.239668:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829435.239671:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d25a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646829824:192.168.203.143@tcp:8:kworker.0 00000100:00000001:3.0:1787829435.239674:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829435.239675:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829435.239676:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829435.239677:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829435.239678:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829435.239679:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829435.239680:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829435.239681:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829435.239682:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829435.239684:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829435.239685:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829435.239686:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829435.239687:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829435.239687:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829435.239689:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829435.239689:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829435.239690:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829435.239691:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829435.239691:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829435.239692:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829435.239693:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829435.239694:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829435.239695:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829435.239695:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829435.239696:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829435.239697:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829435.239698:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829435.239699:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829435.239700:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829435.239701:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829435.239702:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829435.239703:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829435.239705:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787829435.239706:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829435.239708:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1787829435.239709:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829436.134694:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829436.134700:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829436.134767:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829436.134772:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829436.134775:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829437.158649:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829437.158654:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829437.158717:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829437.158721:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829437.158724:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829438.182694:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829438.182699:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829438.182761:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829438.182766:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829438.182770:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829439.206693:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829439.206699:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829439.206774:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829439.206780:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829439.206785:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829440.230694:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829440.230702:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829440.230760:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829440.230766:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829440.230771:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787829440.358773:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829440.358785:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1787829440.358788:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787829440.358791:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787829440.358796:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787829440.358804:0:40798:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 4240 00000100:00080000:3.0:1787829440.358815:0:40798:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787829440.358818:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829440.358820:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:3.0:1787829440.358825:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1787829440.358828:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1787829440.358830:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:3.0:1787829440.358832:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:3.0:1787829440.358834:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829440.358835:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:3.0:1787829440.358838:0:40798:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:3.0:1787829440.358840:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829440.358842:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787829440.358845:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829440.358848:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829440.358850:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:3.0:1787829440.358852:0:40798:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829440.358862:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787829440.358864:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829440.358866:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787829440.358868:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787829440.358872:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829440.358881:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829440.358887:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c7016d00 x1874670646829952/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787829440.358896:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787829440.358899:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829440.358918:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787829440.358920:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829440.358923:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1787829440.358924:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787829440.358925:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787829440.358927:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787829440.358929:0:40798:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4240 00000100:00080000:3.0:1787829440.358933:0:40798:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787829440.358934:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829440.358936:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:3.0:1787829440.358938:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1787829440.358940:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1787829440.358941:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000020:00000001:3.0:1787829440.358943:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:3.0:1787829440.358945:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829440.358946:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:3.0:1787829440.358948:0:40798:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:3.0:1787829440.358949:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829440.358951:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1787829440.358955:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:3.0:1787829440.358955:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829440.358964:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829440.358966:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787829440.358967:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:1.0:1787829440.358979:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000008:00100000:3.0:1787829440.358980:0:40798:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00000001:1.0:1787829440.358983:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000008:00000001:3.0:1787829440.358985:0:40798:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829440.358987:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829440.358989:0:40798:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829440.359000:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1787829440.359003:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787829440.359004:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829440.359010:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1787829440.359012:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829440.359013:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1787829440.359021:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829440.359021:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829440.359054:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829440.359055:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c7016d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646829952:192.168.203.143@tcp:250:kworker.0 00000100:00000001:1.0:1787829440.359063:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00100000:3.0:1787829440.359064:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c70170c0 x1874670646830080/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:1.0:1787829440.359070:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:3.0:1787829440.359072:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 02000000:00000001:1.0:1787829440.359074:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829440.359078:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829440.359081:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:3.0:1787829440.359084:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787829440.359087:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 02000000:00000001:1.0:1787829440.359091:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787829440.359091:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1787829440.359094:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787829440.359098:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787829440.359100:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787829440.359103:0:40798:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4240 00000100:00080000:3.0:1787829440.359106:0:40798:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787829440.359108:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829440.359109:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:3.0:1787829440.359110:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1787829440.359111:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1787829440.359113:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:1.0:1787829440.359115:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000020:00000001:3.0:1787829440.359115:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:1.0:1787829440.359119:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369547008 : -126234340004608 : ffff8d30c7016d00) 00000100:00000001:3.0:1787829440.359122:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787829440.359126:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:3.0:1787829440.359127:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:3.0:1787829440.359135:0:40798:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:3.0:1787829440.359137:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829440.359140:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787829440.359143:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829440.359146:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829440.359148:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:3.0:1787829440.359151:0:40798:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1787829440.359153:0:40798:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829440.359156:0:40798:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829440.359162:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787829440.359163:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829440.359165:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787829440.359167:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787829440.359170:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:1.0:1787829440.359174:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:3.0:1787829440.359177:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829440.359185:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c7017840 x1874670646830208/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787829440.359191:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787829440.359193:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787829440.359196:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829440.359198:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829440.359199:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829440.359203:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787829440.359204:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1787829440.359210:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00080000:3.0:1787829440.359217:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1787829440.359219:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:1.0:1787829440.359222:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787829440.359223:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787829440.359238:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:1.0:1787829440.359239:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4298914560 00000100:00080000:3.0:1787829440.359249:0:40798:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4240 00000100:00080000:3.0:1787829440.359253:0:40798:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1787829440.359255:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 00000100:00000001:3.0:1787829440.359255:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829440.359260:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:3.0:1787829440.359265:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1787829440.359267:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1787829440.359268:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:1.0:1787829440.359270:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4298914560 00000020:00000001:3.0:1787829440.359272:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:3.0:1787829440.359275:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787829440.359276:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 00000100:00000001:3.0:1787829440.359280:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:1.0:1787829440.359284:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00080000:3.0:1787829440.359289:0:40798:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:1.0:1787829440.359294:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829440.359294:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829440.359312:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1787829440.359313:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 02000000:00000001:3.0:1787829440.359324:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829440.359333:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829440.359334:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829440/real 1787829440] req@ffff8d30c7016d00 x1874670646829952/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829465 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829440.359340:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:3.0:1787829440.359344:0:40798:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1787829440.359352:0:40798:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829440.359359:0:40798:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829440.359369:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787829440.359371:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829440.359373:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000400:00000001:1.0:1787829440.359374:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787829440.359379:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787829440.359383:0:2390:0:(events.c:78:reply_in_callback()) Process entered 02000000:00000001:3.0:1787829440.359384:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787829440.359389:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829440.359393:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829440.359398:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c7014000 x1874670646830336/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787829440.359403:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829440.359408:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787829440.359409:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829440.359415:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829440.359416:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829440.359418:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829440.359420:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787829440.359435:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c7016d00 x1874670646829952/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829465 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787829440.359443:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829440.359446:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787829440.359449:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829440.359451:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829440.359453:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829440.359455:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787829440.359458:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787829440.359461:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829440.359467:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787829440.359471:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787829440.359475:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c7016d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646829952:192.168.203.143@tcp:250:kworker.0 00000100:00000001:1.0:1787829440.359481:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829440.359484:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829440.359486:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787829440.359489:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787829440.359493:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787829440.359497:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787829440.359500:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787829440.359506:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787829440.359508:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787829440.359512:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787829440.359513:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829440.359515:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829440.359523:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787829440.359525:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829440.359527:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787829440.359528:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1787829440.359529:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787829440.359531:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787829440.359532:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829440.359534:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787829440.359536:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829440.359538:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829440.359540:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c70170c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646830080:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787829440.359543:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787829440.359545:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787829440.359546:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829440.359547:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787829440.359550:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829440.359556:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787829440.359557:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369547968 : -126234340003648 : ffff8d30c70170c0) 00000100:00000001:1.0:1787829440.359561:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1787829440.359573:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787829440.359577:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787829440.359579:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829440.359580:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829440.359581:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787829440.359583:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829440.359587:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298913302 00000100:00000001:1.0:1787829440.359590:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298914560 00000100:00000001:1.0:1787829440.359592:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298913302 00000100:00000001:1.0:1787829440.359593:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298914560 00000100:00000001:1.0:1787829440.359595:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829440.359596:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829440.359597:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787829440.359621:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829440/real 1787829440] req@ffff8d30c70170c0 x1874670646830080/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829465 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787829440.359627:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787829440.359627:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787829440.359629:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787829440.359630:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787829440.359631:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829440.359633:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829440.359634:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787829440.359637:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c70170c0 x1874670646830080/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829465 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787829440.359642:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829440.359644:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787829440.359646:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829440.359647:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829440.359647:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829440.359649:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787829440.359650:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787829440.359651:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829440.359653:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787829440.359655:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787829440.359657:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c70170c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646830080:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787829440.359661:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787829440.359662:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787829440.359663:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829440.359665:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787829440.359666:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829440.359667:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829440.359669:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c7017840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646830208:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787829440.359672:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787829440.359673:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787829440.359675:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829440.359676:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787829440.359680:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829440.359683:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787829440.359684:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369549888 : -126234340001728 : ffff8d30c7017840) 00000100:00000001:1.0:1787829440.359687:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1787829440.359694:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787829440.359697:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787829440.359698:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829440.359699:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829440.359701:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787829440.359703:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829440.359705:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298913303 00000100:00000001:1.0:1787829440.359707:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298914560 00000100:00000001:1.0:1787829440.359709:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298913303 00000100:00000001:1.0:1787829440.359710:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298914560 00000100:00000001:1.0:1787829440.359712:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829440.359713:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829440.359714:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787829440.359717:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829440/real 1787829440] req@ffff8d30c7017840 x1874670646830208/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829465 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787829440.359727:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787829440.359728:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787829440.359729:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787829440.359730:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787829440.359731:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829440.359732:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829440.359733:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787829440.359735:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c7017840 x1874670646830208/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829465 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787829440.359739:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829440.359740:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787829440.359742:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829440.359743:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829440.359744:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829440.359745:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787829440.359746:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787829440.359748:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829440.359751:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787829440.359753:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787829440.359756:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c7017840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646830208:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787829440.359761:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787829440.359762:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787829440.359764:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829440.359766:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787829440.359768:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829440.359770:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829440.359783:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c7014000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646830336:192.168.203.143@tcp:8:kworker.0 00000100:00000001:1.0:1787829440.359787:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787829440.359790:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787829440.359792:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829440.359794:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787829440.359798:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829440.359804:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787829440.359806:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369535488 : -126234340016128 : ffff8d30c7014000) 00000100:00000001:1.0:1787829440.359809:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1787829440.359823:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787829440.359826:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787829440.359828:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829440.359829:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829440.359830:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787829440.359832:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829440.359835:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298913316 00000100:00000001:1.0:1787829440.359836:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298914560 00000100:00000001:1.0:1787829440.359838:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298913316 00000100:00000001:1.0:1787829440.359840:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298914560 00000100:00000001:1.0:1787829440.359841:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829440.359843:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829440.359844:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787829440.359847:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829440/real 1787829440] req@ffff8d30c7014000 x1874670646830336/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829465 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787829440.359851:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787829440.359852:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787829440.359854:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787829440.359855:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787829440.359856:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829440.359857:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829440.359858:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787829440.359861:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c7014000 x1874670646830336/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829465 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787829440.359865:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829440.359867:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787829440.359869:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829440.359870:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829440.359871:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829440.359872:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787829440.359873:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787829440.359875:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829440.359877:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787829440.359878:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787829440.359880:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c7014000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646830336:192.168.203.143@tcp:8:kworker.0 00000100:00000001:1.0:1787829440.359882:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829440.359884:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829440.359885:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787829440.359886:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787829440.359887:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787829440.359888:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787829440.359889:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787829440.359890:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787829440.359891:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787829440.359892:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787829440.359893:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829440.359894:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829440.359895:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787829440.359896:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787829440.359897:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787829440.359898:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787829440.359899:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787829440.359901:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787829440.359901:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787829440.359902:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787829440.359903:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829440.359904:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829440.359905:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787829440.359905:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787829440.359907:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787829440.359908:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787829440.359909:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787829440.359910:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787829440.359911:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787829440.359912:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787829440.359913:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829440.359914:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829440.359917:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787829440.359918:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829440.359919:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787829440.359921:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829441.254771:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829441.254782:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829441.254864:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829441.254871:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829441.254876:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829442.278688:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829442.278694:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829442.278757:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829442.278761:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829442.278764:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829443.302670:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829443.302675:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829443.302753:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829443.302757:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829443.302760:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829444.326690:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829444.326696:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829444.326754:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829444.326757:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829444.326760:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829445.350715:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829445.350724:0:815:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1787829445.350727:0:815:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1787829445.350730:0:815:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829445.350736:0:815:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1787829445.350739:0:815:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829445.350741:0:815:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829445.350744:0:815:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829445.350807:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829445.350812:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829445.350816:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787829445.478676:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829445.478685:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1787829445.478687:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787829445.478691:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787829445.478694:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787829445.478699:0:40798:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 4245 00000100:00080000:3.0:1787829445.478707:0:40798:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787829445.478709:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829445.478711:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:3.0:1787829445.478719:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1787829445.478721:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1787829445.478724:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:3.0:1787829445.478726:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:3.0:1787829445.478727:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829445.478728:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:3.0:1787829445.478732:0:40798:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:3.0:1787829445.478733:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829445.478736:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787829445.478739:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829445.478742:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829445.478744:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:3.0:1787829445.478747:0:40798:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829445.478759:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787829445.478761:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829445.478765:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787829445.478767:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787829445.478772:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829445.478782:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829445.478792:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c7017c00 x1874670646830464/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787829445.478811:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787829445.478815:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829445.478831:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787829445.478834:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829445.478840:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1787829445.478841:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787829445.478843:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787829445.478845:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787829445.478847:0:40798:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4245 00000100:00080000:3.0:1787829445.478850:0:40798:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787829445.478852:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829445.478853:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:3.0:1787829445.478855:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1787829445.478856:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1787829445.478858:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000020:00000001:3.0:1787829445.478859:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:3.0:1787829445.478860:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829445.478862:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:3.0:1787829445.478865:0:40798:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:3.0:1787829445.478866:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829445.478868:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787829445.478873:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829445.478874:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829445.478876:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:3.0:1787829445.478880:0:40798:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1787829445.478882:0:40798:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829445.478884:0:40798:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829445.478888:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787829445.478889:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829445.478891:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787829445.478892:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787829445.478893:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829445.478900:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829445.478901:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00100000:3.0:1787829445.478907:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c7016940 x1874670646830592/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787829445.478911:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1787829445.478918:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787829445.478925:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787829445.478930:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829445.478933:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787829445.478939:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829445.478943:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:3.0:1787829445.478943:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829445.478949:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00080000:3.0:1787829445.478950:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829445.478962:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 02000000:00000001:1.0:1787829445.478963:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829445.478965:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787829445.478970:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787829445.478973:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787829445.478978:0:40798:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4245 00000100:00100000:1.0:1787829445.478979:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c7017c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646830464:192.168.203.143@tcp:250:kworker.0 00000100:00080000:3.0:1787829445.478982:0:40798:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787829445.478985:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829445.478988:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:3.0:1787829445.478992:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1787829445.478994:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1787829445.478995:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:1.0:1787829445.478997:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000020:00000001:3.0:1787829445.478998:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:3.0:1787829445.479016:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829445.479018:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 02000000:00000001:1.0:1787829445.479020:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000100:00080000:3.0:1787829445.479024:0:40798:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 02000000:00000001:1.0:1787829445.479028:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829445.479028:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829445.479035:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787829445.479040:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829445.479041:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1787829445.479043:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829445.479046:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:3.0:1787829445.479048:0:40798:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1787829445.479050:0:40798:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829445.479051:0:40798:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829445.479053:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829445.479056:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787829445.479059:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829445.479061:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787829445.479062:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787829445.479063:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829445.479069:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829445.479073:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c7014000 x1874670646830720/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787829445.479078:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787829445.479079:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787829445.479084:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369550848 : -126234340000768 : ffff8d30c7017c00) 00000100:00000001:3.0:1787829445.479084:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829445.479091:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787829445.479094:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1787829445.479095:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00080000:3.0:1787829445.479097:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1787829445.479100:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787829445.479102:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787829445.479104:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787829445.479106:0:40798:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4245 00000100:00080000:3.0:1787829445.479109:0:40798:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787829445.479111:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829445.479112:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:3.0:1787829445.479113:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1787829445.479114:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1787829445.479116:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000020:00000001:3.0:1787829445.479118:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:3.0:1787829445.479119:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829445.479122:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:3.0:1787829445.479124:0:40798:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:3.0:1787829445.479125:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829445.479126:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787829445.479128:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829445.479129:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829445.479131:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:3.0:1787829445.479132:0:40798:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1787829445.479134:0:40798:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829445.479137:0:40798:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829445.479142:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787829445.479143:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829445.479144:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787829445.479145:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787829445.479146:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829445.479149:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:1.0:1787829445.479157:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00100000:3.0:1787829445.479161:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c7017840 x1874670646830848/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787829445.479169:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787829445.479172:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829445.479174:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829445.479179:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787829445.479188:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829445.479192:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829445.479195:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787829445.479202:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829445.479208:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4298919680 00000100:00000001:1.0:1787829445.479219:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 00000100:00000001:1.0:1787829445.479240:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4298919680 00000100:00000001:1.0:1787829445.479245:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 00000100:00000001:1.0:1787829445.479249:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829445.479253:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829445.479256:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787829445.479264:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829445/real 1787829445] req@ffff8d30c7017c00 x1874670646830464/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829470 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787829445.479280:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787829445.479282:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787829445.479286:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787829445.479290:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787829445.479292:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829445.479295:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829445.479297:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787829445.479302:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c7017c00 x1874670646830464/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829470 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787829445.479312:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829445.479315:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787829445.479318:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829445.479320:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829445.479321:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829445.479324:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787829445.479327:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787829445.479331:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829445.479335:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787829445.479338:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787829445.479342:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c7017c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646830464:192.168.203.143@tcp:250:kworker.0 00000100:00000001:1.0:1787829445.479348:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829445.479351:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829445.479353:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787829445.479356:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787829445.479360:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787829445.479363:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787829445.479365:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787829445.479370:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787829445.479372:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787829445.479375:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787829445.479376:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829445.479378:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829445.479386:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787829445.479388:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829445.479390:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787829445.479391:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1787829445.479392:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787829445.479393:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787829445.479395:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829445.479397:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787829445.479399:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829445.479400:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829445.479403:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c7016940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646830592:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787829445.479406:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787829445.479407:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787829445.479409:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829445.479410:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787829445.479412:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829445.479417:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787829445.479418:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369546048 : -126234340005568 : ffff8d30c7016940) 00000100:00000001:1.0:1787829445.479423:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1787829445.479433:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787829445.479436:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787829445.479438:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829445.479439:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829445.479440:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787829445.479442:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829445.479446:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298918422 00000100:00000001:1.0:1787829445.479448:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298919680 00000100:00000001:1.0:1787829445.479450:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298918422 00000100:00000001:1.0:1787829445.479452:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298919680 00000100:00000001:1.0:1787829445.479453:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829445.479455:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829445.479456:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787829445.479458:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829445/real 1787829445] req@ffff8d30c7016940 x1874670646830592/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829470 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787829445.479463:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787829445.479464:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787829445.479465:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787829445.479467:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787829445.479468:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829445.479469:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829445.479470:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787829445.479472:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c7016940 x1874670646830592/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829470 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787829445.479476:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829445.479478:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787829445.479479:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829445.479480:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829445.479481:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829445.479482:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787829445.479483:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787829445.479485:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829445.479487:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787829445.479488:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787829445.479490:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c7016940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646830592:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787829445.479495:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787829445.479496:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787829445.479497:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829445.479499:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787829445.479500:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829445.479501:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829445.479503:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c7014000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646830720:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787829445.479505:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787829445.479506:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787829445.479507:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829445.479509:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787829445.479511:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829445.479514:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787829445.479515:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369535488 : -126234340016128 : ffff8d30c7014000) 00000100:00000001:1.0:1787829445.479517:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1787829445.479523:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787829445.479526:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787829445.479527:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829445.479528:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829445.479529:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787829445.479531:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829445.479533:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298918422 00000100:00000001:1.0:1787829445.479535:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298919680 00000100:00000001:1.0:1787829445.479536:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298918422 00000100:00000001:1.0:1787829445.479538:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298919680 00000100:00000001:1.0:1787829445.479540:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829445.479541:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829445.479542:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787829445.479544:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829445/real 1787829445] req@ffff8d30c7014000 x1874670646830720/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829470 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787829445.479548:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787829445.479550:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787829445.479551:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787829445.479553:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787829445.479554:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829445.479556:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829445.479557:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787829445.479561:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c7014000 x1874670646830720/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829470 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787829445.479568:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829445.479571:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787829445.479573:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829445.479575:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829445.479576:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829445.479578:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787829445.479580:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787829445.479582:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829445.479586:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787829445.479588:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787829445.479592:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c7014000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646830720:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787829445.479596:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787829445.479598:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787829445.479637:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829445.479640:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787829445.479642:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829445.479645:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829445.479649:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c7017840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646830848:192.168.203.143@tcp:8:kworker.0 00000100:00000001:1.0:1787829445.479652:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787829445.479654:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787829445.479656:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829445.479658:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787829445.479662:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829445.479668:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787829445.479670:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369549888 : -126234340001728 : ffff8d30c7017840) 00000100:00000001:1.0:1787829445.479673:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1787829445.479687:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787829445.479693:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787829445.479696:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829445.479697:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829445.479699:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787829445.479702:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829445.479706:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298918436 00000100:00000001:1.0:1787829445.479709:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298919680 00000100:00000001:1.0:1787829445.479711:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298918436 00000100:00000001:1.0:1787829445.479713:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298919680 00000100:00000001:1.0:1787829445.479715:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829445.479717:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829445.479719:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787829445.479724:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829445/real 1787829445] req@ffff8d30c7017840 x1874670646830848/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829470 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787829445.479734:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787829445.479736:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787829445.479739:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787829445.479741:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787829445.479742:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829445.479744:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829445.479745:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787829445.479750:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c7017840 x1874670646830848/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829470 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787829445.479758:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829445.479760:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787829445.479763:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829445.479765:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829445.479766:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829445.479768:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787829445.479770:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787829445.479773:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829445.479777:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787829445.479780:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787829445.479784:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c7017840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646830848:192.168.203.143@tcp:8:kworker.0 00000100:00000001:1.0:1787829445.479788:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829445.479790:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829445.479791:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787829445.479793:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787829445.479796:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787829445.479797:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787829445.479799:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787829445.479802:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787829445.479803:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787829445.479806:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787829445.479808:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829445.479810:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829445.479811:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787829445.479812:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787829445.479814:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787829445.479815:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787829445.479817:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787829445.479819:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787829445.479820:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787829445.479822:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787829445.479823:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829445.479825:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829445.479826:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787829445.479828:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787829445.479829:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787829445.479831:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787829445.479832:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787829445.479834:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787829445.479835:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787829445.479837:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787829445.479838:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829445.479840:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829445.479844:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787829445.479846:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829445.479848:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787829445.479850:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829446.374723:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829446.374730:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829446.374785:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829446.374789:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829446.374792:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829447.398706:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829447.398711:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829447.398803:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829447.398807:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829447.398810:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829448.422713:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829448.422719:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829448.422817:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829448.422878:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829448.422881:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829449.446717:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829449.446724:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829449.446793:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829449.446799:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829449.446803:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829450.470700:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829450.470705:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829450.470756:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829450.470759:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829450.470762:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787829450.598692:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829450.598700:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1787829450.598702:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787829450.598705:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787829450.598709:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787829450.598715:0:468:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 4251 00000100:00080000:2.0:1787829450.598723:0:468:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787829450.598725:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829450.598727:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787829450.598730:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787829450.598732:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787829450.598735:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:2.0:1787829450.598736:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:2.0:1787829450.598738:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829450.598739:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:2.0:1787829450.598742:0:468:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787829450.598744:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829450.598746:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787829450.598748:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787829450.598751:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829450.598752:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:2.0:1787829450.598754:0:468:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829450.598762:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787829450.598763:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829450.598766:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787829450.598768:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787829450.598771:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829450.598780:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787829450.598787:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2e1c0 x1874670646830976/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787829450.598795:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787829450.598798:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829450.598809:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787829450.598811:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829450.598814:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1787829450.598815:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787829450.598817:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787829450.598819:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787829450.598820:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4251 00000100:00080000:2.0:1787829450.598823:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787829450.598825:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829450.598826:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787829450.598828:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787829450.598829:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787829450.598831:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000020:00000001:2.0:1787829450.598832:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:2.0:1787829450.598834:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829450.598835:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:2.0:1787829450.598837:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787829450.598839:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829450.598840:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1787829450.598843:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:2.0:1787829450.598845:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829450.598848:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 02000000:00000001:2.0:1787829450.598848:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829450.598849:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:1.0:1787829450.598852:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000008:00100000:2.0:1787829450.598854:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00000001:1.0:1787829450.598855:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787829450.598859:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:2.0:1787829450.598859:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829450.598864:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829450.598865:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787829450.598867:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1787829450.598869:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787829450.598870:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829450.598876:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829450.598876:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787829450.598877:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787829450.598880:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829450.598883:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646830976:192.168.203.143@tcp:250:kworker.0 00000100:00000001:2.0:1787829450.598885:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829450.598889:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00100000:2.0:1787829450.598893:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2d680 x1874670646831104/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:1.0:1787829450.598894:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787829450.598898:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829450.598903:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:2.0:1787829450.598906:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787829450.598908:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829450.598909:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829450.598911:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787829450.598914:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829450.598916:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1787829450.598917:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787829450.598918:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787829450.598920:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787829450.598922:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4251 00000100:00000001:1.0:1787829450.598923:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00080000:2.0:1787829450.598925:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1787829450.598926:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417876928 : -126234291674688 : ffff8d30c9e2e1c0) 00000100:00000001:2.0:1787829450.598930:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787829450.598932:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:2.0:1787829450.598933:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787829450.598936:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787829450.598936:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787829450.598938:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000020:00000001:2.0:1787829450.598939:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:2.0:1787829450.598940:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829450.598941:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:2.0:1787829450.598943:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787829450.598945:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829450.598947:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787829450.598949:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787829450.598950:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829450.598951:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1787829450.598953:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1787829450.598954:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829450.598956:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829450.598960:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787829450.598961:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829450.598962:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787829450.598962:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787829450.598978:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829450.598983:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:1.0:1787829450.598986:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00100000:2.0:1787829450.598990:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2cb40 x1874670646831232/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787829450.599001:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787829450.599002:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1787829450.599003:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829450.599007:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829450.599009:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00080000:2.0:1787829450.599009:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1787829450.599014:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787829450.599015:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1787829450.599018:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1787829450.599020:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787829450.599024:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787829450.599026:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829450.599028:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:1.0:1787829450.599031:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4298924800 00000100:00080000:2.0:1787829450.599034:0:468:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4251 00000100:00000001:1.0:1787829450.599042:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 00000100:00080000:2.0:1787829450.599042:0:468:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787829450.599048:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787829450.599049:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4298924800 00000100:00000001:2.0:1787829450.599051:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:1.0:1787829450.599054:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 00000020:00000001:2.0:1787829450.599056:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:1.0:1787829450.599058:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000020:00000001:2.0:1787829450.599060:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:1.0:1787829450.599061:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829450.599063:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787829450.599068:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829450/real 1787829450] req@ffff8d30c9e2e1c0 x1874670646830976/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829475 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:2.0:1787829450.599072:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000020:00000001:2.0:1787829450.599079:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000400:00000001:1.0:1787829450.599082:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000001:2.0:1787829450.599083:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000001:1.0:1787829450.599084:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1787829450.599087:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:1.0:1787829450.599089:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00080000:2.0:1787829450.599095:0:468:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787829450.599099:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829450.599101:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1787829450.599102:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787829450.599105:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:2.0:1787829450.599105:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829450.599107:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829450.599110:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:2.0:1787829450.599111:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829450.599116:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2e1c0 x1874670646830976/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829475 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787829450.599117:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1787829450.599122:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00000001:1.0:1787829450.599129:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000008:00000001:2.0:1787829450.599130:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829450.599133:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787829450.599134:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829450.599138:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829450.599140:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829450.599141:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:2.0:1787829450.599143:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:1.0:1787829450.599144:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 02000000:00000001:2.0:1787829450.599147:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829450.599148:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787829450.599152:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00080000:1.0:1787829450.599153:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 02000000:00000001:2.0:1787829450.599154:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787829450.599158:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1787829450.599160:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787829450.599164:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000001:2.0:1787829450.599164:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829450.599168:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646830976:192.168.203.143@tcp:250:kworker.0 00000100:00100000:2.0:1787829450.599174:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2d2c0 x1874670646831360/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787829450.599175:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829450.599180:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829450.599182:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787829450.599185:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 00000100:00000001:2.0:1787829450.599186:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 02000000:00000001:1.0:1787829450.599189:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00000001:2.0:1787829450.599190:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829450.599193:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000100:00000001:2.0:1787829450.599194:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829450.599196:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787829450.599201:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787829450.599202:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787829450.599205:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787829450.599206:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829450.599208:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829450.599215:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787829450.599217:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829450.599218:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787829450.599220:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1787829450.599221:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787829450.599222:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787829450.599223:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829450.599224:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787829450.599225:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829450.599227:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829450.599230:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2d680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646831104:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787829450.599232:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787829450.599234:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787829450.599235:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829450.599236:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787829450.599238:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829450.599242:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787829450.599243:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417874048 : -126234291677568 : ffff8d30c9e2d680) 00000100:00000001:1.0:1787829450.599247:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1787829450.599256:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787829450.599259:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787829450.599261:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829450.599262:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829450.599263:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787829450.599265:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829450.599268:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298923542 00000100:00000001:1.0:1787829450.599270:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298924800 00000100:00000001:1.0:1787829450.599272:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298923542 00000100:00000001:1.0:1787829450.599274:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298924800 00000100:00000001:1.0:1787829450.599275:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829450.599276:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829450.599278:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787829450.599280:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829450/real 1787829450] req@ffff8d30c9e2d680 x1874670646831104/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829475 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787829450.599288:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787829450.599289:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787829450.599291:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787829450.599293:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787829450.599294:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829450.599296:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829450.599298:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787829450.599302:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2d680 x1874670646831104/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829475 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787829450.599310:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829450.599312:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787829450.599315:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829450.599317:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829450.599318:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829450.599320:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787829450.599322:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787829450.599325:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829450.599328:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787829450.599331:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787829450.599334:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2d680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646831104:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787829450.599338:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787829450.599339:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787829450.599341:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829450.599343:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787829450.599345:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829450.599347:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829450.599350:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646831232:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787829450.599354:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787829450.599356:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787829450.599357:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829450.599359:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787829450.599362:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829450.599367:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787829450.599369:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417871168 : -126234291680448 : ffff8d30c9e2cb40) 00000100:00000001:1.0:1787829450.599373:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1787829450.599385:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787829450.599388:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787829450.599390:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829450.599391:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829450.599392:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787829450.599394:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829450.599397:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298923542 00000100:00000001:1.0:1787829450.599398:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298924800 00000100:00000001:1.0:1787829450.599400:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298923542 00000100:00000001:1.0:1787829450.599401:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298924800 00000100:00000001:1.0:1787829450.599403:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829450.599404:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829450.599405:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787829450.599408:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829450/real 1787829450] req@ffff8d30c9e2cb40 x1874670646831232/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829475 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787829450.599423:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787829450.599424:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787829450.599426:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787829450.599428:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787829450.599429:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829450.599431:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829450.599433:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787829450.599437:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2cb40 x1874670646831232/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829475 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787829450.599445:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829450.599448:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787829450.599451:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829450.599452:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829450.599454:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829450.599456:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787829450.599458:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787829450.599460:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829450.599464:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787829450.599466:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787829450.599469:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646831232:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787829450.599472:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787829450.599473:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787829450.599474:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829450.599475:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787829450.599477:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829450.599478:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829450.599480:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2d2c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646831360:192.168.203.143@tcp:8:kworker.0 00000100:00000001:1.0:1787829450.599482:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787829450.599483:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787829450.599484:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829450.599486:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787829450.599488:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829450.599492:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787829450.599493:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417873088 : -126234291678528 : ffff8d30c9e2d2c0) 00000100:00000001:1.0:1787829450.599496:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1787829450.599505:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787829450.599508:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787829450.599509:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829450.599510:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829450.599512:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787829450.599514:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829450.599516:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298923555 00000100:00000001:1.0:1787829450.599518:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298924800 00000100:00000001:1.0:1787829450.599519:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298923555 00000100:00000001:1.0:1787829450.599521:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298924800 00000100:00000001:1.0:1787829450.599522:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829450.599523:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829450.599524:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787829450.599527:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829450/real 1787829450] req@ffff8d30c9e2d2c0 x1874670646831360/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829475 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787829450.599531:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787829450.599532:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787829450.599533:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787829450.599534:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787829450.599535:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829450.599536:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829450.599537:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787829450.599540:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2d2c0 x1874670646831360/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829475 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787829450.599544:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829450.599545:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787829450.599546:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829450.599547:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829450.599548:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829450.599549:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787829450.599550:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787829450.599552:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829450.599554:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787829450.599555:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787829450.599557:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2d2c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646831360:192.168.203.143@tcp:8:kworker.0 00000100:00000001:1.0:1787829450.599560:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829450.599561:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829450.599562:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787829450.599563:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787829450.599564:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787829450.599565:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787829450.599566:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787829450.599567:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787829450.599568:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787829450.599570:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787829450.599571:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829450.599572:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829450.599573:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787829450.599573:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787829450.599575:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787829450.599576:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787829450.599576:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787829450.599577:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787829450.599578:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787829450.599580:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787829450.599581:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829450.599582:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829450.599582:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787829450.599583:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787829450.599584:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787829450.599585:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787829450.599586:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787829450.599586:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787829450.599587:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787829450.599588:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787829450.599589:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829450.599590:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829450.599592:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787829450.599593:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829450.599594:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787829450.599595:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829451.494705:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829451.494711:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829451.494769:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829451.494774:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829451.494777:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829452.518699:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829452.518704:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829452.518757:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829452.518760:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829452.518763:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829453.542710:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829453.542717:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829453.542766:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829453.542770:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829453.542773:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829454.566672:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829454.566680:0:815:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1787829454.566683:0:815:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1787829454.566686:0:815:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829454.566692:0:815:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1787829454.566694:0:815:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829454.566696:0:815:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829454.566699:0:815:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829454.566787:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829454.566792:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829454.566796:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829455.590715:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829455.590722:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829455.590790:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829455.590795:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829455.590802:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787829455.718709:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829455.718716:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1787829455.718718:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787829455.718722:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787829455.718726:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787829455.718734:0:468:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 4256 00000100:00080000:3.0:1787829455.718746:0:468:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787829455.718749:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829455.718751:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:3.0:1787829455.718756:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1787829455.718758:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1787829455.718762:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:3.0:1787829455.718765:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:3.0:1787829455.718767:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829455.718769:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:3.0:1787829455.718781:0:468:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:3.0:1787829455.718783:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829455.718785:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787829455.718788:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829455.718791:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829455.718793:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:3.0:1787829455.718795:0:468:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829455.718805:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787829455.718806:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829455.718809:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787829455.718811:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787829455.718814:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829455.718823:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829455.718830:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c70170c0 x1874670646831488/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787829455.718838:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787829455.718841:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829455.718852:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787829455.718854:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829455.718860:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1787829455.718861:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787829455.718862:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787829455.718864:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787829455.718866:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4256 00000100:00080000:3.0:1787829455.718869:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787829455.718870:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829455.718872:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:3.0:1787829455.718873:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1787829455.718875:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1787829455.718877:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000020:00000001:3.0:1787829455.718879:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:3.0:1787829455.718881:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829455.718882:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:3.0:1787829455.718885:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:3.0:1787829455.718886:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829455.718887:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1787829455.718888:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:3.0:1787829455.718892:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829455.718894:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 02000000:00000001:3.0:1787829455.718895:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829455.718897:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787829455.718898:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:1.0:1787829455.718901:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000008:00100000:3.0:1787829455.718903:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00000001:1.0:1787829455.718904:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:3.0:1787829455.718909:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829455.718910:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:3.0:1787829455.718912:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829455.718913:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829455.718916:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1787829455.718918:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829455.718919:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829455.718923:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787829455.718925:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00100000:1.0:1787829455.718926:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c70170c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646831488:192.168.203.143@tcp:250:kworker.0 02000000:00000001:3.0:1787829455.718927:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829455.718943:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:3.0:1787829455.718946:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829455.718949:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787829455.718952:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829455.718955:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c7016d00 x1874670646831616/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:1.0:1787829455.718956:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787829455.718962:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829455.718964:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787829455.718967:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829455.718970:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787829455.718972:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829455.718974:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:1.0:1787829455.718976:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787829455.718976:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787829455.718977:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787829455.718979:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369547968 : -126234340003648 : ffff8d30c70170c0) 00000100:00000001:3.0:1787829455.718984:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:1.0:1787829455.718985:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00080000:3.0:1787829455.718989:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4256 00000100:00080000:3.0:1787829455.718993:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787829455.718995:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829455.718996:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:3.0:1787829455.718997:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1787829455.718998:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1787829455.718999:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000020:00000001:3.0:1787829455.719001:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:3.0:1787829455.719002:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829455.719004:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:3.0:1787829455.719006:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:3.0:1787829455.719007:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829455.719009:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787829455.719012:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829455.719014:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829455.719016:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:3.0:1787829455.719018:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1787829455.719020:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829455.719024:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:1.0:1787829455.719026:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 02000000:00000001:3.0:1787829455.719030:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787829455.719034:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829455.719036:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787829455.719038:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:1.0:1787829455.719040:0:2390:0:(events.c:34:request_out_callback()) Process entered 02000000:00000001:3.0:1787829455.719040:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829455.719046:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829455.719049:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829455.719051:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829455.719055:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00100000:3.0:1787829455.719058:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c7014b40 x1874670646831744/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787829455.719063:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829455.719070:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4298929920 00000100:00000001:3.0:1787829455.719071:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787829455.719075:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829455.719078:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787829455.719080:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1787829455.719083:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 00000100:00080000:3.0:1787829455.719084:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1787829455.719088:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787829455.719090:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787829455.719091:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4298929920 00000100:00000001:1.0:1787829455.719096:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 00000100:00000001:3.0:1787829455.719098:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:1.0:1787829455.719100:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829455.719104:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787829455.719105:0:468:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4256 00000100:00000001:1.0:1787829455.719108:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00080000:3.0:1787829455.719113:0:468:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787829455.719118:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00100000:1.0:1787829455.719119:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829455/real 1787829455] req@ffff8d30c70170c0 x1874670646831488/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829480 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829455.719121:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:3.0:1787829455.719127:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1787829455.719130:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1787829455.719133:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000020:00000001:3.0:1787829455.719137:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:3.0:1787829455.719140:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829455.719142:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000400:00000001:1.0:1787829455.719143:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00080000:3.0:1787829455.719147:0:468:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000400:00000001:1.0:1787829455.719148:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829455.719149:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829455.719152:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1787829455.719154:0:2390:0:(events.c:78:reply_in_callback()) Process entered 02000000:00000001:3.0:1787829455.719155:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829455.719159:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829455.719161:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829455.719161:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:1.0:1787829455.719166:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000008:00100000:3.0:1787829455.719166:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1787829455.719173:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829455.719174:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829455.719175:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829455.719178:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:3.0:1787829455.719181:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787829455.719184:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829455.719186:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c70170c0 x1874670646831488/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829480 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829455.719187:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787829455.719190:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787829455.719192:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829455.719199:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829455.719200:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829455.719206:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00100000:3.0:1787829455.719209:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c70152c0 x1874670646831872/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787829455.719210:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829455.719215:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829455.719217:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829455.719221:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787829455.719224:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829455.719225:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00080000:1.0:1787829455.719228:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00000001:3.0:1787829455.719229:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829455.719234:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1787829455.719235:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787829455.719238:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787829455.719242:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c70170c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646831488:192.168.203.143@tcp:250:kworker.0 00000100:00000001:1.0:1787829455.719246:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829455.719247:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829455.719249:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787829455.719251:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787829455.719255:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787829455.719257:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787829455.719259:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787829455.719263:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787829455.719265:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787829455.719268:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787829455.719269:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829455.719271:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829455.719285:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787829455.719288:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829455.719290:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787829455.719292:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1787829455.719293:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787829455.719296:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787829455.719297:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829455.719300:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787829455.719302:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829455.719304:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829455.719308:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c7016d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646831616:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787829455.719312:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787829455.719314:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787829455.719315:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829455.719317:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787829455.719320:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829455.719327:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787829455.719329:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369547008 : -126234340004608 : ffff8d30c7016d00) 00000100:00000001:1.0:1787829455.719333:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1787829455.719348:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787829455.719353:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787829455.719356:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829455.719357:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829455.719359:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787829455.719362:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829455.719367:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298928662 00000100:00000001:1.0:1787829455.719369:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298929920 00000100:00000001:1.0:1787829455.719371:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298928662 00000100:00000001:1.0:1787829455.719372:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298929920 00000100:00000001:1.0:1787829455.719374:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829455.719376:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829455.719377:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787829455.719381:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829455/real 1787829455] req@ffff8d30c7016d00 x1874670646831616/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829480 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787829455.719391:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787829455.719393:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787829455.719395:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787829455.719397:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787829455.719399:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829455.719400:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829455.719402:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787829455.719406:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c7016d00 x1874670646831616/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829480 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787829455.719413:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829455.719416:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787829455.719418:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829455.719420:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829455.719421:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829455.719423:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787829455.719426:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787829455.719428:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829455.719432:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787829455.719434:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787829455.719447:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c7016d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646831616:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787829455.719452:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787829455.719453:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787829455.719455:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829455.719457:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787829455.719459:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829455.719460:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829455.719462:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c7014b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646831744:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787829455.719465:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787829455.719466:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787829455.719468:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829455.719469:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787829455.719471:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829455.719475:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787829455.719476:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369538368 : -126234340013248 : ffff8d30c7014b40) 00000100:00000001:1.0:1787829455.719479:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1787829455.719489:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787829455.719492:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787829455.719494:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829455.719495:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829455.719496:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787829455.719498:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829455.719501:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298928662 00000100:00000001:1.0:1787829455.719502:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298929920 00000100:00000001:1.0:1787829455.719504:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298928662 00000100:00000001:1.0:1787829455.719505:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298929920 00000100:00000001:1.0:1787829455.719507:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829455.719508:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829455.719509:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787829455.719512:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829455/real 1787829455] req@ffff8d30c7014b40 x1874670646831744/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829480 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787829455.719517:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787829455.719518:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787829455.719519:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787829455.719520:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787829455.719521:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829455.719522:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829455.719523:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787829455.719525:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c7014b40 x1874670646831744/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829480 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787829455.719535:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829455.719536:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787829455.719538:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829455.719539:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829455.719540:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829455.719541:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787829455.719542:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787829455.719543:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829455.719545:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787829455.719546:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787829455.719548:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c7014b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646831744:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787829455.719550:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787829455.719551:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787829455.719552:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829455.719554:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787829455.719555:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829455.719556:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829455.719557:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c70152c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646831872:192.168.203.143@tcp:8:kworker.0 00000100:00000001:1.0:1787829455.719559:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787829455.719560:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787829455.719561:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829455.719562:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787829455.719564:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829455.719567:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787829455.719568:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369540288 : -126234340011328 : ffff8d30c70152c0) 00000100:00000001:1.0:1787829455.719570:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1787829455.719576:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787829455.719579:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787829455.719580:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829455.719581:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829455.719582:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787829455.719584:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829455.719586:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298928675 00000100:00000001:1.0:1787829455.719588:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298929920 00000100:00000001:1.0:1787829455.719590:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298928675 00000100:00000001:1.0:1787829455.719592:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298929920 00000100:00000001:1.0:1787829455.719595:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829455.719597:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829455.719599:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787829455.719635:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829455/real 1787829455] req@ffff8d30c70152c0 x1874670646831872/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829480 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787829455.719644:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787829455.719653:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787829455.719657:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787829455.719658:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787829455.719659:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829455.719660:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829455.719661:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787829455.719665:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c70152c0 x1874670646831872/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829480 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787829455.719669:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829455.719671:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787829455.719673:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829455.719673:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829455.719674:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829455.719675:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787829455.719676:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787829455.719678:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829455.719680:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787829455.719681:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787829455.719684:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c70152c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646831872:192.168.203.143@tcp:8:kworker.0 00000100:00000001:1.0:1787829455.719686:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829455.719688:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829455.719689:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787829455.719690:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787829455.719691:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787829455.719692:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787829455.719693:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787829455.719696:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787829455.719697:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787829455.719699:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787829455.719701:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829455.719702:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829455.719704:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787829455.719705:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787829455.719707:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787829455.719708:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787829455.719709:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787829455.719711:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787829455.719712:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787829455.719713:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787829455.719715:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829455.719716:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829455.719718:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787829455.719719:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787829455.719720:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787829455.719722:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787829455.719723:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787829455.719725:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787829455.719726:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787829455.719727:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787829455.719729:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829455.719731:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829455.719734:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787829455.719736:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829455.719738:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787829455.719739:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829456.614699:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829456.614704:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829456.614764:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829456.614769:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829456.614773:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829457.638670:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829457.638675:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829457.638732:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829457.638737:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829457.638740:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829458.662673:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829458.662679:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829458.662765:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829458.662771:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829458.662774:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829459.686704:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829459.686711:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829459.686764:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829459.686767:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829459.686770:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829460.710722:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829460.710728:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829460.710791:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829460.710795:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829460.710799:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787829460.838724:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829460.838734:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1787829460.838737:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787829460.838741:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787829460.838746:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787829460.838753:0:468:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 4261 00000100:00080000:3.0:1787829460.838765:0:468:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787829460.838768:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829460.838770:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:3.0:1787829460.838776:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1787829460.838779:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1787829460.838782:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:3.0:1787829460.838785:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:3.0:1787829460.838788:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829460.838789:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:3.0:1787829460.838795:0:468:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:3.0:1787829460.838798:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829460.838801:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787829460.838806:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829460.838809:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829460.838813:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:3.0:1787829460.838816:0:468:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829460.838831:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787829460.838834:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829460.838838:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787829460.838842:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787829460.838847:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829460.838859:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829460.838869:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c7015680 x1874670646832000/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787829460.838879:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787829460.838883:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829460.838912:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787829460.838914:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829460.838917:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1787829460.838918:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787829460.838920:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787829460.838922:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787829460.838924:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4261 00000100:00080000:3.0:1787829460.838927:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787829460.838928:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829460.838929:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:3.0:1787829460.838932:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1787829460.838933:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1787829460.838935:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000020:00000001:3.0:1787829460.838942:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:3.0:1787829460.838944:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829460.838945:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:3.0:1787829460.838948:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:3.0:1787829460.838950:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829460.838952:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1787829460.838953:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:3.0:1787829460.838958:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829460.838960:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 02000000:00000001:3.0:1787829460.838961:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829460.838964:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787829460.838965:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:1.0:1787829460.838969:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000008:00100000:3.0:1787829460.838971:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00000001:1.0:1787829460.838972:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:3.0:1787829460.838978:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829460.838981:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:3.0:1787829460.838981:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829460.838985:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829460.838987:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1787829460.838988:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829460.838991:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829460.838995:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00100000:1.0:1787829460.838996:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c7015680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646832000:192.168.203.143@tcp:250:kworker.0 02000000:00000001:3.0:1787829460.838997:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787829460.839000:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829460.839004:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:3.0:1787829460.839007:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829460.839009:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787829460.839012:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829460.839013:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c70143c0 x1874670646832128/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:1.0:1787829460.839015:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787829460.839022:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829460.839022:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787829460.839026:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829460.839030:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787829460.839032:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829460.839034:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1787829460.839035:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787829460.839036:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787829460.839038:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787829460.839040:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4261 00000100:00000001:1.0:1787829460.839042:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00080000:3.0:1787829460.839043:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1787829460.839045:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369541248 : -126234340010368 : ffff8d30c7015680) 00000100:00000001:3.0:1787829460.839047:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787829460.839050:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:3.0:1787829460.839050:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:3.0:1787829460.839055:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1787829460.839056:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1787829460.839058:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000020:00000001:3.0:1787829460.839059:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:3.0:1787829460.839061:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829460.839061:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:3.0:1787829460.839064:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:3.0:1787829460.839065:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829460.839066:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787829460.839068:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829460.839070:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829460.839072:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:3.0:1787829460.839074:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1787829460.839075:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829460.839077:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829460.839081:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787829460.839082:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829460.839083:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787829460.839084:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787829460.839085:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:1.0:1787829460.839094:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:3.0:1787829460.839096:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829460.839102:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c7014f00 x1874670646832256/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787829460.839106:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787829460.839108:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787829460.839110:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829460.839112:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787829460.839113:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829460.839115:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1787829460.839116:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787829460.839119:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787829460.839121:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:1.0:1787829460.839122:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00080000:3.0:1787829460.839125:0:468:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4261 00000100:00000001:1.0:1787829460.839126:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829460.839130:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00080000:3.0:1787829460.839130:0:468:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787829460.839134:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829460.839135:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:3.0:1787829460.839137:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:1.0:1787829460.839138:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829460.839139:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1787829460.839143:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000020:00000001:3.0:1787829460.839145:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:1.0:1787829460.839146:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4298935040 00000100:00000001:3.0:1787829460.839148:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829460.839151:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:3.0:1787829460.839155:0:468:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:3.0:1787829460.839157:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829460.839159:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 02000000:00000001:3.0:1787829460.839160:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787829460.839166:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829460.839169:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4298935040 02000000:00000001:3.0:1787829460.839170:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829460.839176:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:1.0:1787829460.839177:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 00000008:00100000:3.0:1787829460.839180:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00000001:1.0:1787829460.839181:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000008:00000001:3.0:1787829460.839187:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829460.839189:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829460.839191:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829460.839196:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 02000000:00000001:3.0:1787829460.839196:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787829460.839200:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829460.839203:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00100000:1.0:1787829460.839204:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829460/real 1787829460] req@ffff8d30c7015680 x1874670646832000/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829485 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1787829460.839204:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787829460.839207:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829460.839213:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829460.839221:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c70161c0 x1874670646832384/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000001:1.0:1787829460.839226:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000001:3.0:1787829460.839228:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000400:00000001:1.0:1787829460.839230:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829460.839231:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829460.839235:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829460.839237:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787829460.839243:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787829460.839245:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829460.839249:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829460.839251:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787829460.839257:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c7015680 x1874670646832000/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829485 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787829460.839264:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829460.839268:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787829460.839270:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829460.839271:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829460.839272:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829460.839274:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787829460.839275:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787829460.839277:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829460.839281:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787829460.839282:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787829460.839285:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c7015680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646832000:192.168.203.143@tcp:250:kworker.0 00000100:00000001:1.0:1787829460.839291:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829460.839294:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829460.839295:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787829460.839298:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787829460.839302:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787829460.839304:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787829460.839307:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787829460.839312:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787829460.839314:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787829460.839317:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787829460.839319:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829460.839323:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829460.839333:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787829460.839335:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829460.839337:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787829460.839338:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1787829460.839339:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787829460.839340:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787829460.839341:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829460.839343:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787829460.839345:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829460.839346:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829460.839349:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c70143c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646832128:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787829460.839351:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787829460.839353:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787829460.839354:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829460.839356:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787829460.839358:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829460.839363:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787829460.839364:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369536448 : -126234340015168 : ffff8d30c70143c0) 00000100:00000001:1.0:1787829460.839369:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1787829460.839378:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787829460.839382:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787829460.839383:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829460.839384:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829460.839386:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787829460.839388:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829460.839391:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298933782 00000100:00000001:1.0:1787829460.839394:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298935040 00000100:00000001:1.0:1787829460.839397:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298933782 00000100:00000001:1.0:1787829460.839407:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298935040 00000100:00000001:1.0:1787829460.839411:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829460.839413:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829460.839415:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787829460.839420:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829460/real 1787829460] req@ffff8d30c70143c0 x1874670646832128/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829485 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787829460.839431:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787829460.839433:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787829460.839436:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787829460.839438:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787829460.839439:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829460.839441:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829460.839443:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787829460.839447:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c70143c0 x1874670646832128/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829485 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787829460.839452:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829460.839454:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787829460.839455:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829460.839456:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829460.839457:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829460.839458:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787829460.839460:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787829460.839461:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829460.839463:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787829460.839465:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787829460.839468:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c70143c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646832128:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787829460.839472:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787829460.839474:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787829460.839476:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829460.839478:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787829460.839480:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829460.839482:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829460.839485:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c7014f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646832256:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787829460.839489:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787829460.839491:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787829460.839493:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829460.839495:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787829460.839498:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829460.839504:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787829460.839505:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369539328 : -126234340012288 : ffff8d30c7014f00) 00000100:00000001:1.0:1787829460.839509:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1787829460.839520:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787829460.839523:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787829460.839525:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829460.839526:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829460.839527:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787829460.839529:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829460.839532:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298933782 00000100:00000001:1.0:1787829460.839534:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298935040 00000100:00000001:1.0:1787829460.839535:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298933782 00000100:00000001:1.0:1787829460.839537:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298935040 00000100:00000001:1.0:1787829460.839539:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829460.839540:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829460.839541:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787829460.839544:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829460/real 1787829460] req@ffff8d30c7014f00 x1874670646832256/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829485 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787829460.839549:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787829460.839550:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787829460.839551:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787829460.839552:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787829460.839553:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829460.839554:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829460.839556:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787829460.839558:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c7014f00 x1874670646832256/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829485 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787829460.839563:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829460.839564:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787829460.839566:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829460.839567:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829460.839568:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829460.839569:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787829460.839571:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787829460.839572:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829460.839574:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787829460.839575:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787829460.839577:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c7014f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646832256:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787829460.839580:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787829460.839580:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787829460.839581:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829460.839583:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787829460.839584:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829460.839585:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829460.839586:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c70161c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646832384:192.168.203.143@tcp:8:kworker.0 00000100:00000001:1.0:1787829460.839588:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787829460.839589:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787829460.839591:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829460.839592:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787829460.839594:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829460.839598:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787829460.839599:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369544128 : -126234340007488 : ffff8d30c70161c0) 00000100:00000001:1.0:1787829460.839643:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1787829460.839658:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787829460.839662:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787829460.839665:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829460.839666:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829460.839667:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787829460.839670:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829460.839672:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298933796 00000100:00000001:1.0:1787829460.839674:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298935040 00000100:00000001:1.0:1787829460.839676:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298933796 00000100:00000001:1.0:1787829460.839678:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298935040 00000100:00000001:1.0:1787829460.839680:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829460.839681:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829460.839683:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787829460.839688:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829460/real 1787829460] req@ffff8d30c70161c0 x1874670646832384/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829485 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787829460.839697:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787829460.839699:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787829460.839702:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787829460.839703:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787829460.839705:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829460.839707:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829460.839708:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787829460.839711:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c70161c0 x1874670646832384/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829485 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787829460.839718:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829460.839721:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787829460.839723:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829460.839725:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829460.839726:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829460.839727:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787829460.839729:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787829460.839732:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829460.839735:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787829460.839737:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787829460.839740:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c70161c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646832384:192.168.203.143@tcp:8:kworker.0 00000100:00000001:1.0:1787829460.839744:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829460.839746:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829460.839748:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787829460.839750:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787829460.839752:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787829460.839754:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787829460.839755:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787829460.839757:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787829460.839759:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787829460.839761:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787829460.839762:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829460.839764:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829460.839765:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787829460.839766:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787829460.839768:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787829460.839769:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787829460.839770:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787829460.839772:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787829460.839773:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787829460.839775:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787829460.839776:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829460.839778:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829460.839779:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787829460.839781:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787829460.839783:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787829460.839784:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787829460.839785:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787829460.839787:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787829460.839788:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787829460.839789:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787829460.839791:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829460.839792:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829460.839796:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787829460.839798:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829460.839799:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787829460.839801:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829461.734661:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829461.734667:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829461.734762:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829461.734765:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829461.734768:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829462.758711:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829462.758722:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829462.758748:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829462.758751:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829462.758754:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829463.782680:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829463.782688:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829463.782753:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829463.782756:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829463.782759:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829464.806722:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829464.806728:0:815:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1787829464.806730:0:815:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1787829464.806733:0:815:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829464.806737:0:815:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1787829464.806738:0:815:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829464.806740:0:815:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829464.806742:0:815:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829464.806794:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829464.806799:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829464.806804:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829465.830706:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829465.830712:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829465.830770:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829465.830774:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829465.830777:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787829465.958713:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829465.958721:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1787829465.958722:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787829465.958725:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787829465.958729:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787829465.958734:0:468:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 4266 00000100:00080000:3.0:1787829465.958743:0:468:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787829465.958745:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829465.958746:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:3.0:1787829465.958749:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1787829465.958751:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1787829465.958754:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:3.0:1787829465.958755:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:3.0:1787829465.958757:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829465.958758:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:3.0:1787829465.958761:0:468:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:3.0:1787829465.958763:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829465.958765:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787829465.958768:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829465.958770:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829465.958771:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:3.0:1787829465.958773:0:468:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829465.958783:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787829465.958784:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829465.958786:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787829465.958788:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787829465.958792:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829465.958801:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829465.958809:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c7016580 x1874670646832512/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787829465.958817:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787829465.958820:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829465.958834:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787829465.958837:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829465.958839:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1787829465.958840:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787829465.958842:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787829465.958844:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787829465.958845:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4266 00000100:00080000:3.0:1787829465.958847:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787829465.958849:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829465.958850:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:3.0:1787829465.958851:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1787829465.958852:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1787829465.958854:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000020:00000001:3.0:1787829465.958855:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:3.0:1787829465.958871:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829465.958873:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:3.0:1787829465.958876:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:3.0:1787829465.958878:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829465.958880:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787829465.958884:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829465.958886:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829465.958888:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:3.0:1787829465.958891:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1787829465.958893:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829465.958895:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1787829465.958895:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829465.958900:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:1.0:1787829465.958902:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 02000000:00000001:3.0:1787829465.958904:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829465.958907:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787829465.958908:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:1.0:1787829465.958912:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 02000000:00000001:3.0:1787829465.958913:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:1.0:1787829465.958917:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829465.958918:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829465.958926:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:3.0:1787829465.958928:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829465.958931:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829465.958937:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829465.958938:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c7015a40 x1874670646832640/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787829465.958946:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c7016580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646832512:192.168.203.143@tcp:250:kworker.0 00000100:00000001:3.0:1787829465.958955:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787829465.958957:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:3.0:1787829465.958958:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829465.958962:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829465.958963:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000100:00080000:3.0:1787829465.958965:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 02000000:00000001:1.0:1787829465.958968:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787829465.958969:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1787829465.958973:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 02000000:00000001:1.0:1787829465.958974:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00080000:3.0:1787829465.958975:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787829465.958979:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787829465.958981:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4266 02000000:00000001:1.0:1787829465.958982:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787829465.958984:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787829465.958988:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829465.958989:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:3.0:1787829465.958991:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1787829465.958992:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1787829465.958994:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000020:00000001:3.0:1787829465.958995:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:3.0:1787829465.958997:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829465.958999:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:1.0:1787829465.959002:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00080000:3.0:1787829465.959004:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:1.0:1787829465.959006:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369545088 : -126234340006528 : ffff8d30c7016580) 00000100:00000001:3.0:1787829465.959009:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829465.959013:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 02000000:00000001:3.0:1787829465.959014:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787829465.959020:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829465.959023:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829465.959025:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:3.0:1787829465.959028:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1787829465.959031:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829465.959034:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829465.959040:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787829465.959042:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829465.959044:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787829465.959045:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787829465.959047:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829465.959052:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:1.0:1787829465.959055:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00100000:3.0:1787829465.959060:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c70161c0 x1874670646832768/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787829465.959067:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787829465.959069:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829465.959071:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829465.959072:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00080000:3.0:1787829465.959073:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829465.959077:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1787829465.959080:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829465.959080:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787829465.959083:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787829465.959085:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829465.959086:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787829465.959090:0:468:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4266 00000100:00000001:1.0:1787829465.959091:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00080000:3.0:1787829465.959095:0:468:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787829465.959097:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829465.959099:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:1.0:1787829465.959100:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829465.959101:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1787829465.959103:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1787829465.959106:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000020:00000001:3.0:1787829465.959108:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:1.0:1787829465.959109:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4298940160 00000100:00000001:3.0:1787829465.959110:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829465.959113:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:3.0:1787829465.959117:0:468:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:3.0:1787829465.959118:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829465.959119:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1787829465.959122:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 02000000:00000001:3.0:1787829465.959122:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829465.959123:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829465.959128:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:3.0:1787829465.959130:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1787829465.959132:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829465.959133:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4298940160 00000100:00000001:3.0:1787829465.959134:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829465.959142:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:1.0:1787829465.959144:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 02000000:00000001:3.0:1787829465.959145:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829465.959147:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787829465.959150:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787829465.959151:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829465.959152:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829465.959156:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829465.959157:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829465.959161:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c7014f00 x1874670646832896/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787829465.959163:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00000001:3.0:1787829465.959170:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787829465.959172:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829465.959174:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829465/real 1787829465] req@ffff8d30c7016580 x1874670646832512/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829490 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829465.959175:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:1.0:1787829465.959189:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787829465.959192:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787829465.959197:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787829465.959200:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787829465.959202:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829465.959205:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829465.959207:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787829465.959213:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c7016580 x1874670646832512/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829490 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787829465.959222:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829465.959225:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787829465.959228:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829465.959230:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829465.959231:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829465.959234:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787829465.959237:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787829465.959240:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829465.959245:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787829465.959248:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787829465.959251:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c7016580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646832512:192.168.203.143@tcp:250:kworker.0 00000100:00000001:1.0:1787829465.959256:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829465.959258:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829465.959260:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787829465.959262:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787829465.959265:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787829465.959267:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787829465.959269:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787829465.959273:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787829465.959274:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787829465.959277:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787829465.959278:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829465.959280:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829465.959287:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787829465.959289:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829465.959291:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787829465.959293:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1787829465.959295:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787829465.959296:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787829465.959298:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829465.959300:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787829465.959302:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829465.959304:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829465.959308:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c7015a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646832640:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787829465.959311:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787829465.959313:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787829465.959315:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829465.959316:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787829465.959319:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829465.959325:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787829465.959326:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369542208 : -126234340009408 : ffff8d30c7015a40) 00000100:00000001:1.0:1787829465.959330:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1787829465.959342:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787829465.959347:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787829465.959350:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829465.959351:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829465.959353:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787829465.959356:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829465.959360:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298938902 00000100:00000001:1.0:1787829465.959363:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298940160 00000100:00000001:1.0:1787829465.959365:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298938902 00000100:00000001:1.0:1787829465.959368:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298940160 00000100:00000001:1.0:1787829465.959370:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829465.959372:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829465.959374:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787829465.959378:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829465/real 1787829465] req@ffff8d30c7015a40 x1874670646832640/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829490 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787829465.959394:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787829465.959396:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787829465.959398:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787829465.959400:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787829465.959402:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829465.959403:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829465.959405:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787829465.959409:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c7015a40 x1874670646832640/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829490 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787829465.959417:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829465.959420:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787829465.959423:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829465.959424:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829465.959426:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829465.959427:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787829465.959438:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787829465.959441:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829465.959444:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787829465.959446:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787829465.959449:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c7015a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646832640:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787829465.959454:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787829465.959455:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787829465.959457:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829465.959459:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787829465.959461:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829465.959463:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829465.959466:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c70161c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646832768:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787829465.959470:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787829465.959472:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787829465.959474:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829465.959475:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787829465.959479:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829465.959483:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787829465.959485:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369544128 : -126234340007488 : ffff8d30c70161c0) 00000100:00000001:1.0:1787829465.959488:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1787829465.959500:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787829465.959505:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787829465.959507:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829465.959508:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829465.959510:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787829465.959512:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829465.959516:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298938902 00000100:00000001:1.0:1787829465.959519:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298940160 00000100:00000001:1.0:1787829465.959521:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298938902 00000100:00000001:1.0:1787829465.959523:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298940160 00000100:00000001:1.0:1787829465.959525:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829465.959527:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829465.959529:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787829465.959533:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829465/real 1787829465] req@ffff8d30c70161c0 x1874670646832768/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829490 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787829465.959548:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787829465.959550:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787829465.959552:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787829465.959554:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787829465.959556:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829465.959558:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829465.959559:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787829465.959563:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c70161c0 x1874670646832768/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829490 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787829465.959569:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829465.959572:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787829465.959574:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829465.959576:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829465.959584:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829465.959586:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787829465.959588:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787829465.959590:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829465.959594:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787829465.959596:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787829465.959632:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c70161c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646832768:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787829465.959637:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787829465.959638:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787829465.959639:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829465.959642:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787829465.959644:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829465.959646:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829465.959649:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c7014f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646832896:192.168.203.143@tcp:8:kworker.0 00000100:00000001:1.0:1787829465.959652:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787829465.959654:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787829465.959655:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829465.959657:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787829465.959660:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829465.959666:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787829465.959667:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369539328 : -126234340012288 : ffff8d30c7014f00) 00000100:00000001:1.0:1787829465.959670:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1787829465.959681:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787829465.959685:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787829465.959687:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829465.959688:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829465.959690:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787829465.959692:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829465.959696:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298938916 00000100:00000001:1.0:1787829465.959698:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298940160 00000100:00000001:1.0:1787829465.959701:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298938916 00000100:00000001:1.0:1787829465.959703:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298940160 00000100:00000001:1.0:1787829465.959705:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829465.959707:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829465.959709:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787829465.959713:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829465/real 1787829465] req@ffff8d30c7014f00 x1874670646832896/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829490 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787829465.959720:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787829465.959721:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787829465.959724:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787829465.959726:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787829465.959727:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829465.959729:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829465.959731:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787829465.959734:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c7014f00 x1874670646832896/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829490 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787829465.959741:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829465.959743:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787829465.959746:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829465.959747:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829465.959748:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829465.959750:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787829465.959752:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787829465.959755:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829465.959758:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787829465.959760:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787829465.959764:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c7014f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646832896:192.168.203.143@tcp:8:kworker.0 00000100:00000001:1.0:1787829465.959767:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829465.959769:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829465.959771:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787829465.959772:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787829465.959775:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787829465.959776:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787829465.959778:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787829465.959780:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787829465.959781:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787829465.959783:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787829465.959785:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829465.959786:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829465.959788:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787829465.959789:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787829465.959791:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787829465.959792:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787829465.959793:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787829465.959795:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787829465.959796:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787829465.959798:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787829465.959799:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829465.959801:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829465.959802:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787829465.959804:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787829465.959806:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787829465.959808:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787829465.959809:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787829465.959810:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787829465.959811:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787829465.959814:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787829465.959815:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829465.959816:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829465.959820:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787829465.959821:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829465.959823:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787829465.959825:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829466.854673:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829466.854679:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829466.854757:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829466.854762:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829466.854766:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829467.878695:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829467.878700:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829467.878755:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829467.878758:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829467.878761:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829468.902718:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829468.902723:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829468.902790:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829468.902795:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829468.902800:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829469.926692:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829469.926697:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829469.926753:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829469.926760:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829469.926765:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829470.950673:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829470.950679:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829470.950729:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829470.950733:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829470.950736:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787829471.078716:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829471.078728:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1787829471.078731:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787829471.078734:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787829471.078740:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787829471.078747:0:468:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 4271 00000100:00080000:2.0:1787829471.078759:0:468:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787829471.078763:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829471.078765:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787829471.078771:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787829471.078775:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787829471.078780:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:2.0:1787829471.078783:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:2.0:1787829471.078786:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829471.078789:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:2.0:1787829471.078794:0:468:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787829471.078798:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829471.078801:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787829471.078805:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787829471.078809:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829471.078811:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:2.0:1787829471.078814:0:468:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829471.078836:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787829471.078838:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829471.078842:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787829471.078844:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787829471.078848:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829471.078859:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787829471.078870:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2c3c0 x1874670646833024/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787829471.078882:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787829471.078887:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829471.078905:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787829471.078909:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829471.078913:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1787829471.078915:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787829471.078917:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787829471.078921:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787829471.078925:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4271 00000100:00080000:2.0:1787829471.078931:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787829471.078934:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829471.078936:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787829471.078939:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787829471.078942:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:3.0:1787829471.078943:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:2.0:1787829471.078946:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:3.0:1787829471.078950:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000020:00000001:2.0:1787829471.078951:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:3.0:1787829471.078955:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1787829471.078956:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829471.078961:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:3.0:1787829471.078961:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787829471.078966:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787829471.078971:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787829471.078976:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829471.078979:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787829471.078985:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787829471.078988:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829471.078990:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1787829471.078994:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1787829471.078998:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829471.079000:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829471.079005:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787829471.079006:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829471.079008:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787829471.079010:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787829471.079012:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829471.079017:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787829471.079023:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2de00 x1874670646833152/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787829471.079032:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787829471.079034:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829471.079037:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787829471.079040:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829471.079044:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1787829471.079045:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787829471.079047:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787829471.079049:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787829471.079052:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4271 00000100:00080000:2.0:1787829471.079056:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787829471.079059:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829471.079060:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787829471.079063:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787829471.079065:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787829471.079067:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000020:00000001:2.0:1787829471.079069:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:2.0:1787829471.079072:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829471.079073:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:2.0:1787829471.079077:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787829471.079080:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829471.079082:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787829471.079085:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787829471.079087:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829471.079089:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1787829471.079092:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1787829471.079094:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829471.079096:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829471.079100:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787829471.079101:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829471.079103:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787829471.079105:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787829471.079106:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829471.079111:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787829471.079117:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2fc00 x1874670646833280/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787829471.079124:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787829471.079127:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829471.079129:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787829471.079131:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829471.079135:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1787829471.079136:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787829471.079138:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787829471.079141:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787829471.079144:0:468:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4271 00000100:00080000:2.0:1787829471.079148:0:468:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787829471.079150:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829471.079152:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787829471.079155:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787829471.079156:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787829471.079158:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000020:00000001:2.0:1787829471.079160:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:2.0:1787829471.079162:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829471.079164:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:2.0:1787829471.079168:0:468:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787829471.079171:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829471.079173:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787829471.079176:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787829471.079178:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829471.079180:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1787829471.079183:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000008:00000001:2.0:1787829471.079186:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829471.079187:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829471.079192:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787829471.079194:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829471.079195:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:2.0:1787829471.079197:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787829471.079200:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787829471.079201:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787829471.079205:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829471.079207:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829471.079212:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829471.079218:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646833024:192.168.203.143@tcp:250:kworker.0 00000100:00100000:2.0:1787829471.079225:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2c780 x1874670646833408/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787829471.079228:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:2.0:1787829471.079237:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787829471.079240:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829471.079244:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829471.079275:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1787829471.079280:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829471.079284:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1787829471.079289:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829471.079305:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787829471.079307:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417869248 : -126234291682368 : ffff8d30c9e2c3c0) 00000100:00000001:3.0:1787829471.079312:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1787829471.079350:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:3.0:1787829471.079362:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787829471.079366:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829471.079368:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829471.079370:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1787829471.079377:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829471.079381:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4298945280 00000100:00000001:3.0:1787829471.079389:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 00000100:00000001:3.0:1787829471.079395:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4298945280 00000100:00000001:3.0:1787829471.079398:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 00000100:00000001:3.0:1787829471.079402:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829471.079404:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829471.079407:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1787829471.079413:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829471/real 1787829471] req@ffff8d30c9e2c3c0 x1874670646833024/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829496 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1787829471.079425:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787829471.079426:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829471.079430:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829471.079433:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829471.079435:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829471.079438:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829471.079440:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829471.079445:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2c3c0 x1874670646833024/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829496 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829471.079453:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829471.079456:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829471.079460:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829471.079462:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829471.079463:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829471.079465:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829471.079467:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829471.079470:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829471.079475:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829471.079478:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829471.079482:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646833024:192.168.203.143@tcp:250:kworker.0 00000100:00000001:3.0:1787829471.079487:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829471.079489:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829471.079491:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829471.079493:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829471.079497:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829471.079499:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829471.079501:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829471.079504:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829471.079506:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829471.079510:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829471.079511:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829471.079514:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829471.079521:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787829471.079523:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829471.079525:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1787829471.079527:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787829471.079528:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787829471.079530:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787829471.079531:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829471.079534:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787829471.079535:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829471.079538:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829471.079552:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646833152:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829471.079556:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1787829471.079558:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1787829471.079560:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829471.079561:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1787829471.079565:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829471.079571:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787829471.079572:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417875968 : -126234291675648 : ffff8d30c9e2de00) 00000100:00000001:3.0:1787829471.079577:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1787829471.079592:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:3.0:1787829471.079598:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787829471.079627:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829471.079629:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829471.079632:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1787829471.079637:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829471.079643:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298944023 00000100:00000001:3.0:1787829471.079646:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298945280 00000100:00000001:3.0:1787829471.079649:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298944023 00000100:00000001:3.0:1787829471.079652:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298945280 00000100:00000001:3.0:1787829471.079655:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829471.079657:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829471.079660:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1787829471.079666:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829471/real 1787829471] req@ffff8d30c9e2de00 x1874670646833152/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829496 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1787829471.079684:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787829471.079686:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829471.079689:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829471.079691:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829471.079693:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829471.079695:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829471.079696:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829471.079700:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2de00 x1874670646833152/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829496 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829471.079708:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829471.079710:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829471.079713:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829471.079714:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829471.079716:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829471.079718:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829471.079720:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829471.079723:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829471.079726:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829471.079728:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829471.079732:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646833152:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829471.079737:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787829471.079738:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787829471.079740:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829471.079742:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787829471.079744:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829471.079746:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829471.079749:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646833280:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829471.079753:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1787829471.079755:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1787829471.079757:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829471.079759:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1787829471.079762:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829471.079767:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787829471.079769:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417883648 : -126234291667968 : ffff8d30c9e2fc00) 00000100:00000001:3.0:1787829471.079772:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1787829471.079785:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:3.0:1787829471.079790:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787829471.079792:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829471.079794:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829471.079795:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1787829471.079798:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829471.079801:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298944023 00000100:00000001:3.0:1787829471.079804:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298945280 00000100:00000001:3.0:1787829471.079806:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298944023 00000100:00000001:3.0:1787829471.079808:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298945280 00000100:00000001:3.0:1787829471.079811:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829471.079813:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829471.079815:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1787829471.079861:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829471/real 1787829471] req@ffff8d30c9e2fc00 x1874670646833280/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829496 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1787829471.079871:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787829471.079872:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829471.079875:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829471.079877:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829471.079878:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829471.079880:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829471.079881:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829471.079886:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2fc00 x1874670646833280/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829496 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829471.079893:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829471.079895:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829471.079899:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829471.079901:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829471.079902:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829471.079904:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829471.079905:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829471.079908:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829471.079913:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829471.079915:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829471.079918:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646833280:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829471.079922:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787829471.079924:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787829471.079925:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829471.079928:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787829471.079930:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829471.079932:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829471.079935:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646833408:192.168.203.143@tcp:8:kworker.0 00000100:00000001:3.0:1787829471.079939:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1787829471.079941:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1787829471.079943:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829471.079945:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1787829471.079948:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829471.079953:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787829471.079954:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417870208 : -126234291681408 : ffff8d30c9e2c780) 00000100:00000001:3.0:1787829471.079958:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1787829471.079971:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:3.0:1787829471.079975:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787829471.079978:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829471.079979:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829471.079981:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1787829471.079984:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829471.079987:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298944036 00000100:00000001:3.0:1787829471.079990:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298945280 00000100:00000001:3.0:1787829471.079993:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298944036 00000100:00000001:3.0:1787829471.079995:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298945280 00000100:00000001:3.0:1787829471.079997:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829471.079999:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829471.080001:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1787829471.080006:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829471/real 1787829471] req@ffff8d30c9e2c780 x1874670646833408/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829496 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1787829471.080013:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787829471.080014:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829471.080017:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829471.080019:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829471.080020:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829471.080022:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829471.080024:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829471.080028:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2c780 x1874670646833408/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829496 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829471.080035:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829471.080037:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829471.080039:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829471.080041:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829471.080042:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829471.080044:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829471.080045:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829471.080048:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829471.080052:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829471.080054:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829471.080057:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646833408:192.168.203.143@tcp:8:kworker.0 00000100:00000001:3.0:1787829471.080061:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829471.080063:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829471.080065:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829471.080066:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829471.080070:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829471.080072:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829471.080073:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829471.080076:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829471.080077:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829471.080081:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829471.080082:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829471.080084:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829471.080085:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829471.080087:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829471.080089:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829471.080090:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829471.080091:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829471.080093:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829471.080094:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829471.080097:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829471.080099:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829471.080100:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829471.080102:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829471.080103:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829471.080105:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829471.080106:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829471.080107:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829471.080109:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829471.080110:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829471.080112:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829471.080113:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829471.080115:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829471.080118:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787829471.080120:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829471.080122:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1787829471.080123:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829471.974649:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829471.974655:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829471.974712:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829471.974716:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829471.974719:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829472.998708:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829472.998714:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829472.998770:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829472.998773:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829472.998776:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829474.022669:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829474.022675:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829474.022727:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829474.022730:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829474.022733:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829475.046726:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829475.046735:0:815:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1787829475.046738:0:815:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1787829475.046741:0:815:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829475.046747:0:815:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1787829475.046749:0:815:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829475.046751:0:815:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829475.046754:0:815:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829475.046818:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829475.046822:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829475.046824:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829476.070666:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829476.070671:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829476.070724:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829476.070728:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829476.070732:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787829476.198702:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829476.198709:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1787829476.198711:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787829476.198714:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787829476.198718:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787829476.198724:0:468:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 4276 00000100:00080000:2.0:1787829476.198732:0:468:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787829476.198734:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829476.198736:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787829476.198739:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787829476.198741:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787829476.198743:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:2.0:1787829476.198745:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:2.0:1787829476.198747:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829476.198748:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:2.0:1787829476.198751:0:468:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787829476.198752:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829476.198754:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787829476.198758:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787829476.198760:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829476.198762:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:2.0:1787829476.198763:0:468:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829476.198772:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787829476.198774:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829476.198776:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787829476.198778:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787829476.198781:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829476.198801:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787829476.198810:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2f0c0 x1874670646833536/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787829476.198817:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787829476.198821:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829476.198837:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787829476.198840:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829476.198842:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1787829476.198843:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787829476.198845:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787829476.198846:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787829476.198848:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4276 00000100:00080000:2.0:1787829476.198851:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787829476.198852:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829476.198853:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787829476.198855:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787829476.198856:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787829476.198857:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000020:00000001:2.0:1787829476.198859:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:2.0:1787829476.198860:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829476.198861:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:2.0:1787829476.198864:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787829476.198865:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829476.198867:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787829476.198872:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787829476.198874:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829476.198875:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1787829476.198879:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1787829476.198881:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829476.198883:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787829476.198884:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829476.198889:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:3.0:1787829476.198891:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 02000000:00000001:2.0:1787829476.198892:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829476.198896:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787829476.198897:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:3.0:1787829476.198897:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 02000000:00000001:2.0:1787829476.198902:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829476.198902:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787829476.198906:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829476.198907:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787829476.198913:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2ed00 x1874670646833664/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:3.0:1787829476.198914:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787829476.198919:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787829476.198923:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 02000000:00000001:3.0:1787829476.198925:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829476.198926:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829476.198932:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787829476.198936:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1787829476.198937:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646833536:192.168.203.143@tcp:250:kworker.0 00000100:00080000:2.0:1787829476.198939:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1787829476.198942:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787829476.198947:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787829476.198947:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:2.0:1787829476.198953:0:468:0:(import.c:485:import_select_connection()) Process entered 02000000:00000001:3.0:1787829476.198954:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000100:00080000:2.0:1787829476.198959:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4276 02000000:00000001:3.0:1787829476.198960:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787829476.198966:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 02000000:00000001:3.0:1787829476.198966:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1787829476.198971:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829476.198979:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829476.198981:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787829476.198984:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787829476.198986:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:3.0:1787829476.198988:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000020:00000001:2.0:1787829476.198989:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:3.0:1787829476.198992:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417880768 : -126234291670848 : ffff8d30c9e2f0c0) 00000020:00000001:2.0:1787829476.198994:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:3.0:1787829476.198999:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:2.0:1787829476.199000:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829476.199004:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:2.0:1787829476.199009:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787829476.199012:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829476.199014:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787829476.199017:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787829476.199020:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829476.199022:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1787829476.199024:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1787829476.199028:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829476.199030:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829476.199036:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787829476.199038:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829476.199041:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000800:00000100:3.0:1787829476.199043:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 02000000:00000001:2.0:1787829476.199044:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787829476.199049:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829476.199054:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787829476.199060:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2e940 x1874670646833792/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787829476.199060:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787829476.199067:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787829476.199069:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787829476.199072:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829476.199072:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829476.199079:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829476.199079:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00080000:2.0:1787829476.199084:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829476.199086:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1787829476.199088:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:3.0:1787829476.199088:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787829476.199091:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787829476.199094:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:3.0:1787829476.199096:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4298950400 00000100:00080000:2.0:1787829476.199097:0:468:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4276 00000100:00080000:2.0:1787829476.199103:0:468:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787829476.199105:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829476.199106:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787829476.199108:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787829476.199110:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:3.0:1787829476.199110:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 00000020:00000001:2.0:1787829476.199113:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000020:00000001:2.0:1787829476.199116:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:2.0:1787829476.199119:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829476.199119:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4298950400 00000100:00000001:2.0:1787829476.199120:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:2.0:1787829476.199126:0:468:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787829476.199128:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829476.199129:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 02000000:00000001:2.0:1787829476.199130:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787829476.199134:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829476.199136:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787829476.199137:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829476.199141:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1787829476.199144:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00000001:3.0:1787829476.199144:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:2.0:1787829476.199150:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829476.199151:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00000001:2.0:1787829476.199152:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829476.199158:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787829476.199159:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829476.199162:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787829476.199164:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00100000:3.0:1787829476.199164:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829476/real 1787829476] req@ffff8d30c9e2f0c0 x1874670646833536/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829501 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1787829476.199167:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829476.199172:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787829476.199180:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2c000 x1874670646833920/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000001:3.0:1787829476.199186:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787829476.199190:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1787829476.199192:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787829476.199195:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1787829476.199196:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829476.199202:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829476.199202:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829476.199207:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829476.199213:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829476.199215:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829476.199221:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2f0c0 x1874670646833536/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829501 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829476.199230:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829476.199233:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829476.199236:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829476.199239:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829476.199240:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829476.199243:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829476.199244:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829476.199247:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829476.199251:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829476.199254:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829476.199257:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646833536:192.168.203.143@tcp:250:kworker.0 00000100:00000001:3.0:1787829476.199263:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829476.199265:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829476.199266:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829476.199269:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829476.199273:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829476.199275:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829476.199277:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829476.199280:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829476.199282:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829476.199286:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829476.199287:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829476.199289:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829476.199295:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787829476.199297:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829476.199299:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1787829476.199301:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787829476.199302:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787829476.199304:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787829476.199306:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829476.199309:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787829476.199310:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829476.199312:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829476.199315:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646833664:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829476.199319:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1787829476.199321:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1787829476.199322:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829476.199324:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1787829476.199327:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829476.199333:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787829476.199335:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417879808 : -126234291671808 : ffff8d30c9e2ed00) 00000100:00000001:3.0:1787829476.199340:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1787829476.199354:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:3.0:1787829476.199360:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787829476.199363:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829476.199364:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829476.199366:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1787829476.199368:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829476.199373:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298949142 00000100:00000001:3.0:1787829476.199376:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298950400 00000100:00000001:3.0:1787829476.199378:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298949142 00000100:00000001:3.0:1787829476.199380:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298950400 00000100:00000001:3.0:1787829476.199383:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829476.199384:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829476.199386:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1787829476.199391:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829476/real 1787829476] req@ffff8d30c9e2ed00 x1874670646833664/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829501 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1787829476.199400:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787829476.199401:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829476.199404:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829476.199406:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829476.199407:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829476.199409:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829476.199411:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829476.199415:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2ed00 x1874670646833664/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829501 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829476.199422:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829476.199424:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829476.199426:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829476.199428:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829476.199429:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829476.199431:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829476.199433:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829476.199436:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829476.199439:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829476.199442:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829476.199445:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646833664:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829476.199449:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787829476.199451:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787829476.199452:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829476.199454:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787829476.199456:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829476.199458:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829476.199461:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646833792:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829476.199465:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1787829476.199466:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1787829476.199468:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829476.199470:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1787829476.199473:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829476.199478:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787829476.199480:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417878848 : -126234291672768 : ffff8d30c9e2e940) 00000100:00000001:3.0:1787829476.199483:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1787829476.199496:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:3.0:1787829476.199501:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787829476.199504:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829476.199505:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829476.199507:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1787829476.199510:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829476.199513:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298949142 00000100:00000001:3.0:1787829476.199516:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298950400 00000100:00000001:3.0:1787829476.199518:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298949142 00000100:00000001:3.0:1787829476.199521:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298950400 00000100:00000001:3.0:1787829476.199523:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829476.199525:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829476.199527:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1787829476.199531:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829476/real 1787829476] req@ffff8d30c9e2e940 x1874670646833792/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829501 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1787829476.199540:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787829476.199541:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829476.199543:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829476.199545:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829476.199547:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829476.199548:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829476.199550:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829476.199554:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2e940 x1874670646833792/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829501 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829476.199561:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829476.199563:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829476.199566:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829476.199568:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829476.199569:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829476.199571:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829476.199572:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829476.199575:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829476.199579:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829476.199582:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829476.199585:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646833792:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829476.199590:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787829476.199591:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787829476.199593:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829476.199594:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787829476.199596:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829476.199598:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829476.199636:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646833920:192.168.203.143@tcp:8:kworker.0 00000100:00000001:3.0:1787829476.199639:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1787829476.199641:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1787829476.199643:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829476.199645:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1787829476.199649:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829476.199654:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787829476.199655:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417868288 : -126234291683328 : ffff8d30c9e2c000) 00000100:00000001:3.0:1787829476.199659:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1787829476.199672:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:3.0:1787829476.199678:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787829476.199680:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829476.199682:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829476.199683:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1787829476.199686:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829476.199690:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298949156 00000100:00000001:3.0:1787829476.199692:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298950400 00000100:00000001:3.0:1787829476.199695:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298949156 00000100:00000001:3.0:1787829476.199697:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298950400 00000100:00000001:3.0:1787829476.199699:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829476.199701:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829476.199703:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1787829476.199707:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829476/real 1787829476] req@ffff8d30c9e2c000 x1874670646833920/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829501 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1787829476.199721:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787829476.199723:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829476.199725:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829476.199727:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829476.199728:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829476.199730:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829476.199732:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829476.199736:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2c000 x1874670646833920/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829501 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829476.199743:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829476.199745:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829476.199748:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829476.199749:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829476.199751:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829476.199753:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829476.199754:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829476.199757:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829476.199760:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829476.199762:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829476.199766:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646833920:192.168.203.143@tcp:8:kworker.0 00000100:00000001:3.0:1787829476.199770:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829476.199771:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829476.199773:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829476.199774:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829476.199777:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829476.199779:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829476.199780:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829476.199782:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829476.199784:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829476.199802:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829476.199803:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829476.199806:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829476.199807:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829476.199808:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829476.199811:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829476.199812:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829476.199813:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829476.199815:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829476.199816:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829476.199818:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829476.199819:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829476.199821:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829476.199822:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829476.199823:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829476.199825:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829476.199827:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829476.199828:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829476.199830:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829476.199831:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829476.199833:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829476.199834:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829476.199837:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829476.199840:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787829476.199841:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829476.199843:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1787829476.199845:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829477.094717:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829477.094725:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829477.094759:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829477.094763:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829477.094766:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829478.118712:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829478.118719:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829478.118783:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829478.118787:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829478.118790:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829479.142680:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829479.142685:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829479.142726:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829479.142729:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829479.142732:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829480.166716:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829480.166721:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829480.166774:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829480.166777:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829480.166780:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829481.190674:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829481.190680:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829481.190730:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829481.190733:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829481.190735:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787829481.318709:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829481.318717:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1787829481.318720:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787829481.318723:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787829481.318727:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787829481.318734:0:468:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 4281 00000100:00080000:3.0:1787829481.318744:0:468:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787829481.318748:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829481.318760:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:3.0:1787829481.318764:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1787829481.318767:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1787829481.318770:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:3.0:1787829481.318773:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:3.0:1787829481.318775:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829481.318776:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:3.0:1787829481.318781:0:468:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:3.0:1787829481.318783:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829481.318785:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787829481.318789:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829481.318792:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829481.318794:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:3.0:1787829481.318796:0:468:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829481.318806:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787829481.318808:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829481.318811:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787829481.318813:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787829481.318817:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829481.318827:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829481.318835:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c70143c0 x1874670646834048/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787829481.318844:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787829481.318848:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829481.318863:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787829481.318865:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829481.318868:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1787829481.318870:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787829481.318871:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787829481.318874:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787829481.318876:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4281 00000100:00080000:3.0:1787829481.318880:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787829481.318882:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829481.318884:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:3.0:1787829481.318887:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1787829481.318888:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:1.0:1787829481.318891:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:3.0:1787829481.318891:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:1.0:1787829481.318896:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000020:00000001:3.0:1787829481.318896:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:1.0:1787829481.318899:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787829481.318901:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787829481.318902:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787829481.318904:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829481.318904:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 02000000:00000001:1.0:1787829481.318911:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000100:00080000:3.0:1787829481.318912:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 02000000:00000001:1.0:1787829481.318914:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829481.318916:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829481.318918:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829481.318919:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00100000:1.0:1787829481.318923:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c70143c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646834048:192.168.203.143@tcp:250:kworker.0 02000000:00000001:3.0:1787829481.318925:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829481.318927:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1787829481.318930:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829481.318932:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:3.0:1787829481.318934:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 02000000:00000001:1.0:1787829481.318935:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829481.318938:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000008:00100000:3.0:1787829481.318939:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 02000000:00000001:1.0:1787829481.318942:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:3.0:1787829481.318943:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829481.318947:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829481.318952:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787829481.318953:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829481.318955:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787829481.318956:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:1.0:1787829481.318957:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369536448 : -126234340015168 : ffff8d30c70143c0) 00000100:00000001:1.0:1787829481.318961:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 02000000:00000001:3.0:1787829481.318966:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787829481.318970:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829481.318979:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829481.318985:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c7015680 x1874670646834176/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787829481.318990:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:3.0:1787829481.318997:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787829481.319000:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829481.319001:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787829481.319004:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829481.319005:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829481.319009:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787829481.319010:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1787829481.319011:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00080000:3.0:1787829481.319016:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:1.0:1787829481.319017:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829481.319020:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:1.0:1787829481.319021:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4298955520 00000100:00080000:3.0:1787829481.319023:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787829481.319028:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:1.0:1787829481.319029:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 00000100:00080000:3.0:1787829481.319033:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4281 00000100:00000001:1.0:1787829481.319036:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4298955520 00000100:00080000:3.0:1787829481.319038:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1787829481.319039:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 00000100:00000001:1.0:1787829481.319044:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829481.319044:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829481.319047:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:1.0:1787829481.319048:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829481.319052:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000020:00000001:3.0:1787829481.319054:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1787829481.319056:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00100000:1.0:1787829481.319057:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829481/real 1787829481] req@ffff8d30c70143c0 x1874670646834048/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829506 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:3.0:1787829481.319059:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000020:00000001:3.0:1787829481.319065:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:3.0:1787829481.319071:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000001:1.0:1787829481.319072:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000001:3.0:1787829481.319073:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000400:00000001:1.0:1787829481.319074:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787829481.319078:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787829481.319081:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00080000:3.0:1787829481.319081:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:1.0:1787829481.319084:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829481.319086:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829481.319087:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829481.319089:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:3.0:1787829481.319091:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00100000:1.0:1787829481.319094:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c70143c0 x1874670646834048/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829506 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1787829481.319096:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829481.319101:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829481.319103:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829481.319105:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787829481.319107:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829481.319108:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829481.319109:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829481.319111:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:1.0:1787829481.319112:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000008:00100000:3.0:1787829481.319116:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00000001:1.0:1787829481.319117:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787829481.319121:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000008:00000001:3.0:1787829481.319123:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1787829481.319125:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829481.319127:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829481.319128:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787829481.319132:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c70143c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646834048:192.168.203.143@tcp:250:kworker.0 02000000:00000001:3.0:1787829481.319134:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:1.0:1787829481.319136:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829481.319138:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 02000000:00000001:3.0:1787829481.319138:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829481.319141:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 00000100:00000001:3.0:1787829481.319142:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1787829481.319143:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829481.319145:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1787829481.319146:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787829481.319150:0:2390:0:(genops.c:1217:class_import_put()) Process entered 02000000:00000001:3.0:1787829481.319150:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829481.319151:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787829481.319154:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787829481.319156:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787829481.319158:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829481.319158:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829481.319160:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829481.319164:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829481.319169:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00100000:3.0:1787829481.319169:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c70152c0 x1874670646834304/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787829481.319172:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829481.319175:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787829481.319177:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1787829481.319179:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787829481.319181:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787829481.319183:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829481.319185:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 02000000:00000001:1.0:1787829481.319186:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787829481.319188:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829481.319191:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829481.319192:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829481.319198:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c7015680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646834176:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829481.319199:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829481.319203:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00080000:3.0:1787829481.319205:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 02000000:00000001:1.0:1787829481.319206:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000100:00080000:3.0:1787829481.319211:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1787829481.319213:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 02000000:00000001:1.0:1787829481.319214:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787829481.319215:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 02000000:00000001:1.0:1787829481.319216:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787829481.319219:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829481.319221:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:1.0:1787829481.319224:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787829481.319227:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369541248 : -126234340010368 : ffff8d30c7015680) 00000100:00080000:3.0:1787829481.319227:0:468:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4281 00000100:00000001:1.0:1787829481.319232:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00080000:3.0:1787829481.319234:0:468:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787829481.319238:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829481.319240:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000800:00000100:1.0:1787829481.319243:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000020:00000001:3.0:1787829481.319244:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1787829481.319247:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:1.0:1787829481.319248:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000020:00000001:3.0:1787829481.319250:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:1.0:1787829481.319251:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829481.319253:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829481.319255:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000020:00000001:3.0:1787829481.319255:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:1.0:1787829481.319258:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829481.319259:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787829481.319262:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298954262 00000100:00000001:3.0:1787829481.319262:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:1.0:1787829481.319266:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298955520 00000100:00080000:3.0:1787829481.319268:0:468:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:1.0:1787829481.319269:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298954262 00000100:00000001:1.0:1787829481.319273:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298955520 00000100:00000001:3.0:1787829481.319274:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829481.319275:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829481.319279:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829481.319280:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1787829481.319281:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 02000000:00000001:3.0:1787829481.319285:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00100000:1.0:1787829481.319286:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829481/real 1787829481] req@ffff8d30c7015680 x1874670646834176/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829506 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1787829481.319292:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:1.0:1787829481.319295:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787829481.319297:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829481.319297:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:1.0:1787829481.319301:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000008:00100000:3.0:1787829481.319302:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00000001:1.0:1787829481.319303:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787829481.319305:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829481.319309:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000008:00000001:3.0:1787829481.319309:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829481.319310:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829481.319314:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829481.319315:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c7015680 x1874670646834176/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829506 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1787829481.319320:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:1.0:1787829481.319321:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:3.0:1787829481.319323:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829481.319324:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787829481.319329:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829481.319329:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:1.0:1787829481.319333:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 02000000:00000001:3.0:1787829481.319333:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:1.0:1787829481.319335:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:3.0:1787829481.319337:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829481.319338:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787829481.319340:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787829481.319343:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829481.319345:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829481.319345:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829481.319349:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787829481.319352:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c7015680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646834176:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787829481.319356:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829481.319356:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c7014b40 x1874670646834432/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787829481.319359:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829481.319360:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787829481.319362:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787829481.319364:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787829481.319366:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787829481.319367:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787829481.319369:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787829481.319371:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787829481.319372:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829481.319374:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787829481.319375:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829481.319378:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829481.319380:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829481.319381:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787829481.319383:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829481.319386:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829481.319387:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787829481.319390:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1787829481.319393:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787829481.319395:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787829481.319396:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829481.319399:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787829481.319401:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829481.319403:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829481.319407:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c70152c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646834304:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787829481.319410:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787829481.319412:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787829481.319414:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829481.319416:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787829481.319418:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829481.319422:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787829481.319424:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369540288 : -126234340011328 : ffff8d30c70152c0) 00000100:00000001:1.0:1787829481.319427:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1787829481.319438:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787829481.319442:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787829481.319444:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829481.319445:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829481.319447:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787829481.319448:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829481.319451:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298954262 00000100:00000001:1.0:1787829481.319452:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298955520 00000100:00000001:1.0:1787829481.319454:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298954262 00000100:00000001:1.0:1787829481.319455:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298955520 00000100:00000001:1.0:1787829481.319456:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829481.319457:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829481.319458:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787829481.319461:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829481/real 1787829481] req@ffff8d30c70152c0 x1874670646834304/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829506 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787829481.319465:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787829481.319466:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787829481.319467:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787829481.319468:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787829481.319469:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829481.319470:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829481.319471:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787829481.319473:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c70152c0 x1874670646834304/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829506 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787829481.319477:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829481.319478:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787829481.319479:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829481.319480:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829481.319480:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829481.319481:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787829481.319482:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787829481.319483:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829481.319486:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787829481.319487:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787829481.319489:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c70152c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646834304:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787829481.319491:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787829481.319492:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787829481.319492:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829481.319494:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787829481.319494:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829481.319496:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829481.319497:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c7014b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646834432:192.168.203.143@tcp:8:kworker.0 00000100:00000001:1.0:1787829481.319499:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787829481.319500:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787829481.319501:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829481.319502:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787829481.319504:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829481.319507:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787829481.319507:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369538368 : -126234340013248 : ffff8d30c7014b40) 00000100:00000001:1.0:1787829481.319509:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1787829481.319516:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787829481.319518:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787829481.319519:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829481.319520:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829481.319521:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787829481.319522:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829481.319524:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298954275 00000100:00000001:1.0:1787829481.319525:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298955520 00000100:00000001:1.0:1787829481.319527:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298954275 00000100:00000001:1.0:1787829481.319528:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298955520 00000100:00000001:1.0:1787829481.319529:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829481.319530:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829481.319531:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787829481.319533:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829481/real 1787829481] req@ffff8d30c7014b40 x1874670646834432/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829506 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787829481.319538:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787829481.319538:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787829481.319539:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787829481.319540:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787829481.319541:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829481.319542:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829481.319543:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787829481.319545:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c7014b40 x1874670646834432/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829506 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787829481.319549:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829481.319550:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787829481.319551:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829481.319551:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829481.319552:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829481.319553:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787829481.319554:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787829481.319555:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829481.319557:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787829481.319558:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787829481.319559:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c7014b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646834432:192.168.203.143@tcp:8:kworker.0 00000100:00000001:1.0:1787829481.319561:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829481.319562:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829481.319563:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787829481.319563:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787829481.319564:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787829481.319565:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787829481.319566:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787829481.319567:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787829481.319568:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787829481.319569:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787829481.319570:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829481.319571:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829481.319571:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787829481.319572:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787829481.319573:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787829481.319574:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787829481.319575:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787829481.319575:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787829481.319576:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787829481.319577:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787829481.319577:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829481.319578:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829481.319580:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787829481.319581:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829481.319582:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787829481.319582:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829482.214684:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829482.214689:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829482.214742:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829482.214757:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829482.214760:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829483.238666:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829483.238671:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829483.238722:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829483.238725:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829483.238728:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829484.262774:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829484.262783:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829484.262839:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829484.262844:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829484.262847:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829485.286652:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829485.286658:0:815:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1787829485.286659:0:815:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1787829485.286661:0:815:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829485.286665:0:815:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1787829485.286666:0:815:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829485.286668:0:815:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829485.286669:0:815:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829485.286707:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829485.286710:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829485.286712:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829486.310700:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829486.310706:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829486.310777:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829486.310780:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829486.310782:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787829486.438695:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829486.438702:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1787829486.438704:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1787829486.438707:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787829486.438710:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1787829486.438724:0:40701:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 4286 00000100:00080000:1.0:1787829486.438733:0:40701:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1787829486.438735:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787829486.438737:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:1.0:1787829486.438740:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:1.0:1787829486.438742:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:1.0:1787829486.438745:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:1.0:1787829486.438747:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:1.0:1787829486.438748:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787829486.438749:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:1.0:1787829486.438753:0:40701:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:1.0:1787829486.438754:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829486.438756:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1787829486.438758:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829486.438760:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829486.438762:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:1.0:1787829486.438765:0:40701:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829486.438775:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1787829486.438776:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829486.438780:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1787829486.438782:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1787829486.438786:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829486.438797:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829486.438806:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d13ecf00 x1874670646834560/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787829486.438816:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787829486.438820:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829486.438829:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1787829486.438831:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829486.438833:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1787829486.438835:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1787829486.438836:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787829486.438837:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1787829486.438839:0:40701:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4286 00000100:00080000:1.0:1787829486.438842:0:40701:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1787829486.438844:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787829486.438845:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:1.0:1787829486.438846:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:1.0:1787829486.438848:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:1.0:1787829486.438850:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000020:00000001:1.0:1787829486.438851:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:1.0:1787829486.438853:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787829486.438853:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:1.0:1787829486.438855:0:40701:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:1.0:1787829486.438857:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829486.438857:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:1.0:1787829486.438858:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:3.0:1787829486.438863:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 02000000:00000001:1.0:1787829486.438864:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829486.438867:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 02000000:00000001:1.0:1787829486.438868:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829486.438871:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:3.0:1787829486.438872:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000008:00100000:1.0:1787829486.438875:0:40701:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00000001:3.0:1787829486.438876:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:1.0:1787829486.438878:0:40701:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829486.438880:0:40701:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829486.438882:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787829486.438885:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787829486.438886:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829486.438887:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829486.438891:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787829486.438891:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829486.438892:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1787829486.438895:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829486.438899:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829486.438900:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d13ecf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646834560:192.168.203.143@tcp:250:kworker.0 00000100:00100000:1.0:1787829486.438906:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d13eda40 x1874670646834688/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787829486.438908:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1787829486.438913:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:1.0:1787829486.438914:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787829486.438916:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829486.438917:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829486.438919:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829486.438921:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00080000:1.0:1787829486.438922:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829486.438925:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:1.0:1787829486.438926:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1787829486.438928:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 02000000:00000001:3.0:1787829486.438928:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829486.438933:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1787829486.438934:0:40701:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4286 00000100:00080000:1.0:1787829486.438936:0:40701:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1787829486.438938:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787829486.438938:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:1.0:1787829486.438940:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:1.0:1787829486.438940:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:1.0:1787829486.438942:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000020:00000001:1.0:1787829486.438944:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:3.0:1787829486.438944:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787829486.438948:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829486.438948:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839541341952 : -126234168209664 : ffff8d30d13ecf00) 00000100:00000001:1.0:1787829486.438950:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:3.0:1787829486.438955:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00080000:1.0:1787829486.438957:0:40701:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:1.0:1787829486.438961:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829486.438962:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1787829486.438965:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829486.438967:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829486.438968:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:1.0:1787829486.438970:0:40701:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1787829486.438972:0:40701:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829486.438974:0:40701:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829486.438978:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1787829486.438980:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829486.438981:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1787829486.438982:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1787829486.438984:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829486.438990:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:3.0:1787829486.438990:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00100000:1.0:1787829486.438999:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d13ee1c0 x1874670646834816/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787829486.439003:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787829486.439005:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787829486.439007:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829486.439009:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829486.439010:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829486.439013:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1787829486.439014:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1787829486.439018:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00080000:1.0:1787829486.439019:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1787829486.439022:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1787829486.439023:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787829486.439026:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829486.439027:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:3.0:1787829486.439031:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4298960640 00000100:00080000:1.0:1787829486.439032:0:40701:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4286 00000100:00080000:1.0:1787829486.439039:0:40701:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1787829486.439041:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829486.439042:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 00000100:00000001:1.0:1787829486.439043:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:1.0:1787829486.439048:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:3.0:1787829486.439049:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4298960640 00000020:00000001:1.0:1787829486.439050:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:1.0:1787829486.439053:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:3.0:1787829486.439056:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 00000020:00000001:1.0:1787829486.439058:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:3.0:1787829486.439060:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829486.439062:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829486.439065:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829486.439066:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:3.0:1787829486.439070:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00080000:1.0:1787829486.439073:0:40701:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:1.0:1787829486.439079:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829486.439080:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829486/real 1787829486] req@ffff8d30d13ecf00 x1874670646834560/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829511 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:1.0:1787829486.439082:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1787829486.439087:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829486.439092:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829486.439094:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000400:00000001:3.0:1787829486.439095:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000008:00100000:1.0:1787829486.439096:0:40701:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000008:00000001:1.0:1787829486.439099:0:40701:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:3.0:1787829486.439099:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787829486.439103:0:40701:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829486.439105:0:2390:0:(events.c:78:reply_in_callback()) Process entered 02000000:00000001:1.0:1787829486.439107:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:3.0:1787829486.439109:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 02000000:00000001:1.0:1787829486.439110:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829486.439113:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1787829486.439114:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:3.0:1787829486.439114:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:1.0:1787829486.439116:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829486.439118:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829486.439121:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829486.439122:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787829486.439129:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d13ee580 x1874670646834944/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787829486.439131:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d13ecf00 x1874670646834560/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829511 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787829486.439141:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787829486.439145:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829486.439147:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829486.439152:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829486.439156:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829486.439159:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829486.439161:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829486.439163:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829486.439165:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829486.439168:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829486.439170:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829486.439174:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829486.439176:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829486.439180:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d13ecf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646834560:192.168.203.143@tcp:250:kworker.0 00000100:00000001:3.0:1787829486.439184:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829486.439186:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829486.439188:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829486.439190:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829486.439193:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829486.439194:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829486.439197:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829486.439200:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829486.439202:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829486.439205:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829486.439206:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829486.439208:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829486.439215:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787829486.439216:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829486.439218:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1787829486.439219:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787829486.439220:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787829486.439222:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787829486.439223:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829486.439225:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787829486.439226:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829486.439228:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829486.439231:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d13eda40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646834688:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829486.439234:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1787829486.439236:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1787829486.439237:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829486.439239:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1787829486.439241:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829486.439246:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787829486.439247:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839541344832 : -126234168206784 : ffff8d30d13eda40) 00000100:00000001:3.0:1787829486.439251:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1787829486.439263:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:3.0:1787829486.439267:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787829486.439269:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829486.439270:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829486.439271:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1787829486.439274:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829486.439278:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298959382 00000100:00000001:3.0:1787829486.439280:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298960640 00000100:00000001:3.0:1787829486.439283:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298959382 00000100:00000001:3.0:1787829486.439285:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298960640 00000100:00000001:3.0:1787829486.439286:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829486.439288:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829486.439290:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1787829486.439294:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829486/real 1787829486] req@ffff8d30d13eda40 x1874670646834688/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829511 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1787829486.439301:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787829486.439302:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829486.439304:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829486.439306:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829486.439307:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829486.439308:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829486.439309:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829486.439312:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d13eda40 x1874670646834688/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829511 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829486.439319:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829486.439321:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829486.439323:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829486.439324:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829486.439325:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829486.439327:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829486.439329:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829486.439331:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829486.439334:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829486.439336:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829486.439338:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d13eda40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646834688:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829486.439342:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787829486.439343:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787829486.439344:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829486.439345:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787829486.439347:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829486.439348:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829486.439351:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d13ee1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646834816:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829486.439356:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1787829486.439357:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1787829486.439358:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829486.439360:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1787829486.439362:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829486.439366:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787829486.439367:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839541346752 : -126234168204864 : ffff8d30d13ee1c0) 00000100:00000001:3.0:1787829486.439370:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1787829486.439379:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:3.0:1787829486.439382:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787829486.439384:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829486.439385:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829486.439387:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1787829486.439389:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829486.439392:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298959382 00000100:00000001:3.0:1787829486.439394:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298960640 00000100:00000001:3.0:1787829486.439396:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298959382 00000100:00000001:3.0:1787829486.439398:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298960640 00000100:00000001:3.0:1787829486.439400:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829486.439402:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829486.439404:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1787829486.439407:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829486/real 1787829486] req@ffff8d30d13ee1c0 x1874670646834816/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829511 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1787829486.439414:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787829486.439415:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829486.439416:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829486.439418:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829486.439419:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829486.439421:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829486.439422:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829486.439425:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d13ee1c0 x1874670646834816/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829511 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829486.439436:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829486.439438:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829486.439440:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829486.439442:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829486.439443:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829486.439445:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829486.439446:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829486.439448:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829486.439451:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829486.439453:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829486.439456:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d13ee1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646834816:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829486.439459:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787829486.439460:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787829486.439461:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829486.439463:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787829486.439465:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829486.439467:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829486.439469:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d13ee580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646834944:192.168.203.143@tcp:8:kworker.0 00000100:00000001:3.0:1787829486.439472:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1787829486.439474:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1787829486.439475:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829486.439477:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1787829486.439480:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829486.439485:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787829486.439486:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839541347712 : -126234168203904 : ffff8d30d13ee580) 00000100:00000001:3.0:1787829486.439489:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1787829486.439501:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:3.0:1787829486.439505:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787829486.439507:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829486.439508:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829486.439510:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1787829486.439512:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829486.439515:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298959395 00000100:00000001:3.0:1787829486.439517:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298960640 00000100:00000001:3.0:1787829486.439519:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298959395 00000100:00000001:3.0:1787829486.439521:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298960640 00000100:00000001:3.0:1787829486.439523:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829486.439525:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829486.439526:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1787829486.439530:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829486/real 1787829486] req@ffff8d30d13ee580 x1874670646834944/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829511 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1787829486.439535:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787829486.439535:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829486.439537:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829486.439538:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829486.439539:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829486.439540:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829486.439540:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829486.439543:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d13ee580 x1874670646834944/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829511 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829486.439547:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829486.439548:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829486.439550:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829486.439550:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829486.439551:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829486.439552:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829486.439553:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829486.439554:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829486.439556:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829486.439557:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829486.439559:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d13ee580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646834944:192.168.203.143@tcp:8:kworker.0 00000100:00000001:3.0:1787829486.439562:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829486.439563:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829486.439564:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829486.439565:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829486.439566:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829486.439567:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829486.439567:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829486.439568:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829486.439569:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829486.439570:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829486.439571:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829486.439572:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829486.439573:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829486.439573:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829486.439574:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829486.439575:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829486.439576:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829486.439577:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829486.439577:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829486.439578:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829486.439579:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829486.439580:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829486.439580:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829486.439581:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829486.439582:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829486.439582:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829486.439583:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829486.439585:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829486.439585:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829486.439586:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829486.439586:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829486.439587:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829486.439589:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787829486.439590:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829486.439591:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1787829486.439592:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829487.334723:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829487.334729:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829487.334791:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829487.334796:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829487.334799:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829488.358688:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829488.358693:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829488.358767:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829488.358771:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829488.358774:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829489.382669:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829489.382673:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829489.382723:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829489.382727:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829489.382730:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829490.406707:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829490.406714:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829490.406775:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829490.406778:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829490.406781:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829491.430701:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829491.430708:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829491.430766:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829491.430770:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829491.430773:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787829491.558725:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829491.558734:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1787829491.558737:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787829491.558740:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787829491.558744:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787829491.558750:0:40701:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 4292 00000100:00080000:0.0:1787829491.558759:0:40701:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787829491.558762:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829491.558763:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787829491.558767:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787829491.558770:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787829491.558773:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:0.0:1787829491.558775:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:0.0:1787829491.558776:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829491.558777:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:0.0:1787829491.558781:0:40701:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787829491.558784:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829491.558786:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787829491.558790:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787829491.558792:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829491.558794:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1787829491.558796:0:40701:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829491.558807:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787829491.558809:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829491.558811:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787829491.558814:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787829491.558817:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829491.558827:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787829491.558835:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d27c00 x1874670646835072/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787829491.558844:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787829491.558847:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829491.558861:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787829491.558864:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829491.558867:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1787829491.558868:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787829491.558869:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787829491.558872:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787829491.558874:0:40701:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4292 00000100:00080000:0.0:1787829491.558878:0:40701:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787829491.558880:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829491.558881:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787829491.558884:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787829491.558886:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787829491.558888:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000020:00000001:0.0:1787829491.558890:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:3.0:1787829491.558891:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787829491.558892:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829491.558895:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:3.0:1787829491.558896:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787829491.558899:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00080000:0.0:1787829491.558902:0:40701:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:3.0:1787829491.558902:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787829491.558905:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829491.558908:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829491.558911:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787829491.558912:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787829491.558915:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787829491.558920:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829491.558920:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829491.558925:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829491.558927:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00100000:3.0:1787829491.558930:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d27c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646835072:192.168.203.143@tcp:250:kworker.0 00000008:00100000:0.0:1787829491.558931:0:40701:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1787829491.558936:0:40701:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829491.558938:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:0.0:1787829491.558939:0:40701:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829491.558944:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787829491.558945:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829491.558946:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:0.0:1787829491.558947:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787829491.558949:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787829491.558950:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829491.558951:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829491.558957:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:0.0:1787829491.558958:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787829491.558964:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d252c0 x1874670646835200/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:3.0:1787829491.558964:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829491.558972:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787829491.558974:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829491.558976:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787829491.558978:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829491.558980:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1787829491.558981:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:3.0:1787829491.558981:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00080000:0.0:1787829491.558986:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787829491.558986:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601351680 : -126234108199936 : ffff8d30d4d27c00) 00000100:00000001:0.0:1787829491.558990:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787829491.558992:0:40701:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4292 00000100:00000001:3.0:1787829491.558994:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00080000:0.0:1787829491.558996:0:40701:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787829491.558998:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829491.558999:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787829491.559001:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787829491.559002:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787829491.559003:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000020:00000001:0.0:1787829491.559004:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:0.0:1787829491.559005:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829491.559006:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:0.0:1787829491.559008:0:40701:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787829491.559010:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829491.559011:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787829491.559012:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787829491.559014:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829491.559015:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1787829491.559016:0:40701:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1787829491.559018:0:40701:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829491.559019:0:40701:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829491.559022:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787829491.559022:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829491.559023:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787829491.559024:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787829491.559025:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829491.559028:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787829491.559031:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d25680 x1874670646835328/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787829491.559035:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787829491.559036:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829491.559038:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787829491.559039:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829491.559041:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1787829491.559041:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787829491.559043:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787829491.559043:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829491.559048:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787829491.559050:0:40701:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4292 00000100:00000001:3.0:1787829491.559051:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4298965760 00000100:00080000:0.0:1787829491.559053:0:40701:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787829491.559056:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829491.559057:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787829491.559059:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787829491.559059:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787829491.559061:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000020:00000001:0.0:1787829491.559063:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:3.0:1787829491.559063:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1787829491.559066:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829491.559067:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:0.0:1787829491.559069:0:40701:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787829491.559071:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829491.559072:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4298965760 02000000:00000001:0.0:1787829491.559073:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787829491.559075:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787829491.559078:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829491.559079:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1787829491.559081:0:40701:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00000001:3.0:1787829491.559081:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 00000008:00000001:0.0:1787829491.559084:0:40701:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829491.559087:0:40701:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829491.559088:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787829491.559090:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787829491.559093:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829491.559094:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:3.0:1787829491.559094:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829491.559097:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787829491.559099:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829491.559100:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829491.559103:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829491.559103:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787829491.559107:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d26940 x1874670646835456/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787829491.559108:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787829491.559112:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:0.0:1787829491.559113:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787829491.559115:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787829491.559116:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829491.559117:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787829491.559119:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1787829491.559120:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829491.559121:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787829491.559124:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829491.559127:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787829491.559129:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829491.559132:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829491.559136:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d252c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646835200:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829491.559140:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1787829491.559142:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1787829491.559144:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829491.559146:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1787829491.559149:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829491.559156:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787829491.559157:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601341120 : -126234108210496 : ffff8d30d4d252c0) 00000100:00000001:3.0:1787829491.559162:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:3.0:1787829491.559175:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829491.559179:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298964502 00000100:00000001:3.0:1787829491.559181:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298965760 00000100:00000001:3.0:1787829491.559182:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298964502 00000100:00000001:3.0:1787829491.559183:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298965760 00000100:00000001:3.0:1787829491.559185:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829491.559186:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829491.559187:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787829491.559188:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787829491.559188:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829491.559190:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787829491.559191:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829491.559192:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829491.559194:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d25680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646835328:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829491.559196:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1787829491.559197:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1787829491.559198:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829491.559199:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1787829491.559201:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829491.559204:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787829491.559205:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601342080 : -126234108209536 : ffff8d30d4d25680) 00000100:00000001:3.0:1787829491.559207:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:3.0:1787829491.559212:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829491.559214:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298964502 00000100:00000001:3.0:1787829491.559216:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298965760 00000100:00000001:3.0:1787829491.559217:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298964502 00000100:00000001:3.0:1787829491.559218:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298965760 00000100:00000001:3.0:1787829491.559219:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829491.559220:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829491.559221:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829491.559223:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829491.559224:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787829491.559225:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:3.0:1787829491.559226:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1787829491.559227:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787829491.559228:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787829491.559229:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787829491.559230:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829491.559232:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787829491.559233:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829491.559235:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829491.559238:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d26940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646835456:192.168.203.143@tcp:8:kworker.0 00000100:00000001:3.0:1787829491.559240:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1787829491.559242:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1787829491.559243:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829491.559245:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1787829491.559247:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829491.559252:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787829491.559253:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601346880 : -126234108204736 : ffff8d30d4d26940) 00000100:00000001:3.0:1787829491.559256:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:3.0:1787829491.559265:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829491.559269:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298964515 00000100:00000001:3.0:1787829491.559271:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298965760 00000100:00000001:3.0:1787829491.559273:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298964515 00000100:00000001:3.0:1787829491.559276:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298965760 00000100:00000001:3.0:1787829491.559278:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829491.559280:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829491.559282:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829491.559283:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829491.559286:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787829491.559287:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:3.0:1787829491.559289:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1787829491.559290:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787829491.559292:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829491.559294:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829492.454683:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829492.454688:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829492.454744:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829492.454747:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829492.454749:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829493.478682:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829493.478688:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829493.478740:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829493.478744:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829493.478748:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829494.502699:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829494.502705:0:815:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1787829494.502707:0:815:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1787829494.502709:0:815:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829494.502712:0:815:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1787829494.502713:0:815:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829494.502715:0:815:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829494.502716:0:815:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829494.502769:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829494.502772:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829494.502774:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:2.0:1787829494.630800:0:2381:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.143:988 00000400:00000100:2.0:1787829494.630830:0:2381:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.143@tcp at host 192.168.203.143 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1787829494.630841:0:2381:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.143@tcp 00000100:00000001:2.0:1787829494.630851:0:2381:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1787829494.630863:0:2381:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787829494.630865:0:2381:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829494.630867:0:2381:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:2.0:1787829494.630875:0:2381:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.143@tcp 00000100:00000001:2.0:1787829494.630879:0:2381:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1787829494.630881:0:2381:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787829494.630882:0:2381:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829494.630884:0:2381:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:2.0:1787829494.630887:0:2381:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.143@tcp 00000100:00000001:2.0:1787829494.630891:0:2381:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1787829494.630893:0:2381:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787829494.630894:0:2381:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829494.630896:0:2381:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:2.0:1787829494.630899:0:2381:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.143@tcp 00000100:00000001:3.0:1787829494.630899:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787829494.630903:0:2381:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787829494.630903:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1787829494.630905:0:2381:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787829494.630906:0:2381:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829494.630906:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00000001:2.0:1787829494.630907:0:2381:0:(events.c:68:request_out_callback()) Process leaving 00000100:00100000:3.0:1787829494.630922:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829491/real 1787829494] req@ffff8d30d4d252c0 x1874670646835200/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829516 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1787829494.630935:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787829494.630937:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829494.630941:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829494.630943:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829494.630944:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829494.630946:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829494.630948:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829494.630951:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d252c0 x1874670646835200/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829516 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829494.630955:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829494.630958:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829494.630960:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829494.630961:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829494.630961:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829494.630963:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829494.630966:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829494.630968:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829494.630970:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829494.630973:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829494.630976:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d252c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646835200:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829494.630981:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1787829494.630984:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829491/real 1787829494] req@ffff8d30d4d25680 x1874670646835328/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829516 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1787829494.630992:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787829494.630993:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829494.631004:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829494.631006:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829494.631007:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829494.631009:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829494.631010:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829494.631013:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d25680 x1874670646835328/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829516 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829494.631019:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829494.631021:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829494.631034:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829494.631035:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829494.631036:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829494.631037:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829494.631039:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829494.631041:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829494.631045:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829494.631047:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829494.631050:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d25680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646835328:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829494.631064:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1787829494.631068:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829491/real 1787829494] req@ffff8d30d4d27c00 x1874670646835072/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829516 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1787829494.631075:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787829494.631077:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829494.631081:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829494.631082:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829494.631083:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829494.631085:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829494.631086:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829494.631090:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d27c00 x1874670646835072/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829516 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829494.631095:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829494.631097:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829494.631099:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829494.631100:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829494.631101:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829494.631103:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829494.631104:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829494.631106:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829494.631109:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829494.631110:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829494.631113:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d27c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646835072:192.168.203.143@tcp:250:kworker.0 00000100:00000001:3.0:1787829494.631117:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829494.631118:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829494.631120:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829494.631122:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829494.631127:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829494.631130:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829494.631132:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829494.631135:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829494.631137:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829494.631140:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829494.631141:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829494.631143:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829494.631144:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829494.631145:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829494.631147:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829494.631148:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829494.631149:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829494.631151:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829494.631151:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829494.631153:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829494.631154:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829494.631155:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829494.631156:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829494.631157:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829494.631159:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829494.631160:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829494.631161:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829494.631162:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829494.631163:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829494.631165:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829494.631166:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829494.631168:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829494.631170:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1787829494.631171:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787829494.631173:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1787829494.631177:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829491/real 1787829494] req@ffff8d30d4d26940 x1874670646835456/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829516 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1787829494.631184:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787829494.631185:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829494.631188:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829494.631189:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829494.631190:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829494.631191:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829494.631192:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829494.631195:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d26940 x1874670646835456/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829516 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829494.631201:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829494.631203:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829494.631205:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829494.631206:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829494.631207:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829494.631208:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829494.631210:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829494.631212:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829494.631215:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829494.631217:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829494.631220:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d26940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646835456:192.168.203.143@tcp:8:kworker.0 00000100:00000001:3.0:1787829494.631224:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829494.631226:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829494.631227:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829494.631228:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829494.631230:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829494.631231:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829494.631232:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829494.631234:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829494.631235:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829494.631238:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829494.631239:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829494.631241:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829494.631249:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787829494.631251:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829494.631252:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1787829494.631253:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829495.526668:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829495.526673:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829495.526719:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829495.526724:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829495.526727:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829496.550698:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829496.550703:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829496.550751:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829496.550753:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829496.550756:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787829496.678709:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829496.678718:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1787829496.678721:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1787829496.678725:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787829496.678729:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1787829496.678736:0:40701:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 4297 00000100:00080000:1.0:1787829496.678752:0:40701:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1787829496.678756:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787829496.678758:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:1.0:1787829496.678762:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:1.0:1787829496.678764:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:1.0:1787829496.678767:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:1.0:1787829496.678769:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:1.0:1787829496.678771:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787829496.678773:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:1.0:1787829496.678778:0:40701:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:1.0:1787829496.678781:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829496.678783:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1787829496.678786:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829496.678789:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829496.678792:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:1.0:1787829496.678794:0:40701:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829496.678800:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1787829496.678801:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829496.678803:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1787829496.678805:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1787829496.678808:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829496.678817:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829496.678823:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d13ef480 x1874670646835584/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787829496.678829:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787829496.678832:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829496.678853:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1787829496.678856:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829496.678859:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1787829496.678861:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1787829496.678863:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787829496.678865:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1787829496.678867:0:40701:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4297 00000100:00080000:1.0:1787829496.678871:0:40701:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1787829496.678873:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787829496.678875:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:1.0:1787829496.678877:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:1.0:1787829496.678879:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:1.0:1787829496.678881:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000020:00000001:1.0:1787829496.678883:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:1.0:1787829496.678885:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787829496.678886:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:1.0:1787829496.678890:0:40701:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:3.0:1787829496.678891:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787829496.678892:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829496.678896:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:3.0:1787829496.678896:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787829496.678900:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 02000000:00000001:1.0:1787829496.678901:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829496.678902:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787829496.678905:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829496.678906:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829496.678910:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 02000000:00000001:3.0:1787829496.678911:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000008:00100000:1.0:1787829496.678914:0:40701:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 02000000:00000001:3.0:1787829496.678915:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829496.678920:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:1.0:1787829496.678921:0:40701:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829496.678925:0:40701:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829496.678927:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d13ef480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646835584:192.168.203.143@tcp:250:kworker.0 02000000:00000001:1.0:1787829496.678930:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1787829496.678933:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829496.678936:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1787829496.678938:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1787829496.678940:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829496.678944:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:1.0:1787829496.678945:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829496.678949:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000100:00100000:1.0:1787829496.678954:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d13ef840 x1874670646835712/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:3.0:1787829496.678954:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829496.678959:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:1.0:1787829496.678966:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 02000000:00000001:3.0:1787829496.678966:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829496.678970:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829496.678974:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1787829496.678977:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829496.678980:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:1.0:1787829496.678982:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1787829496.678984:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787829496.678986:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1787829496.678989:0:40701:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4297 00000100:00000001:3.0:1787829496.678989:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787829496.678993:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839541351552 : -126234168200064 : ffff8d30d13ef480) 00000100:00080000:1.0:1787829496.678996:0:40701:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787829496.678999:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:1.0:1787829496.679000:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787829496.679003:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:1.0:1787829496.679006:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:1.0:1787829496.679008:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:1.0:1787829496.679010:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000020:00000001:1.0:1787829496.679012:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:1.0:1787829496.679014:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787829496.679016:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:1.0:1787829496.679019:0:40701:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:1.0:1787829496.679022:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829496.679024:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1787829496.679027:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829496.679030:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829496.679032:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:1.0:1787829496.679034:0:40701:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1787829496.679036:0:40701:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829496.679038:0:40701:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:3.0:1787829496.679043:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 02000000:00000001:1.0:1787829496.679044:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1787829496.679047:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829496.679050:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1787829496.679051:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1787829496.679053:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829496.679056:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787829496.679058:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829496.679061:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829496.679064:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829496.679066:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d13eed00 x1874670646835840/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787829496.679067:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787829496.679076:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787829496.679079:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829496.679083:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829496.679083:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1787829496.679089:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1787829496.679091:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4298970880 00000100:00080000:1.0:1787829496.679093:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1787829496.679097:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1787829496.679098:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787829496.679101:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:3.0:1787829496.679104:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 00000100:00080000:1.0:1787829496.679105:0:40701:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4297 00000100:00080000:1.0:1787829496.679111:0:40701:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787829496.679114:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4298970880 00000100:00000001:1.0:1787829496.679115:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787829496.679117:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:3.0:1787829496.679121:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 00000020:00000001:1.0:1787829496.679122:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:1.0:1787829496.679124:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:1.0:1787829496.679127:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:3.0:1787829496.679128:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000020:00000001:1.0:1787829496.679132:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:3.0:1787829496.679134:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829496.679138:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829496.679139:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00000001:1.0:1787829496.679141:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:1.0:1787829496.679148:0:40701:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00100000:3.0:1787829496.679149:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829496/real 1787829496] req@ffff8d30d13ef480 x1874670646835584/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829521 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787829496.679152:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829496.679157:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1787829496.679161:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829496.679165:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829496.679168:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000400:00000001:3.0:1787829496.679168:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787829496.679172:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000008:00100000:1.0:1787829496.679173:0:40701:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000008:00000001:1.0:1787829496.679178:0:40701:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829496.679178:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787829496.679180:0:40701:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829496.679183:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 02000000:00000001:1.0:1787829496.679187:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:3.0:1787829496.679187:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:1.0:1787829496.679190:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829496.679192:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829496.679194:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:3.0:1787829496.679196:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:1.0:1787829496.679197:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1787829496.679201:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829496.679204:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d13ef480 x1874670646835584/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829521 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787829496.679207:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829496.679217:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787829496.679218:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d13ede00 x1874670646835968/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787829496.679224:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829496.679229:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829496.679232:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787829496.679233:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829496.679236:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829496.679237:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829496.679239:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787829496.679241:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829496.679243:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829496.679248:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829496.679259:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829496.679262:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829496.679266:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d13ef480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646835584:192.168.203.143@tcp:250:kworker.0 00000100:00000001:3.0:1787829496.679271:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829496.679273:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829496.679275:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829496.679277:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829496.679280:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829496.679282:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829496.679284:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829496.679288:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829496.679290:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829496.679293:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829496.679294:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829496.679296:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829496.679303:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787829496.679304:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829496.679306:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1787829496.679308:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787829496.679309:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787829496.679310:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787829496.679311:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829496.679313:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787829496.679315:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829496.679317:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829496.679321:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d13ef840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646835712:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829496.679324:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1787829496.679326:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1787829496.679328:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829496.679330:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1787829496.679332:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829496.679338:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787829496.679340:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839541352512 : -126234168199104 : ffff8d30d13ef840) 00000100:00000001:3.0:1787829496.679345:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1787829496.679358:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:3.0:1787829496.679363:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787829496.679366:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829496.679367:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829496.679368:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1787829496.679371:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829496.679375:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298969622 00000100:00000001:3.0:1787829496.679378:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298970880 00000100:00000001:3.0:1787829496.679380:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298969622 00000100:00000001:3.0:1787829496.679382:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298970880 00000100:00000001:3.0:1787829496.679384:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829496.679386:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829496.679388:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1787829496.679393:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829496/real 1787829496] req@ffff8d30d13ef840 x1874670646835712/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829521 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1787829496.679404:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787829496.679405:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829496.679407:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829496.679409:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829496.679410:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829496.679411:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829496.679413:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829496.679416:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d13ef840 x1874670646835712/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829521 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829496.679422:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829496.679424:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829496.679426:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829496.679427:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829496.679428:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829496.679430:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829496.679431:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829496.679433:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829496.679436:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829496.679438:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829496.679441:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d13ef840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646835712:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829496.679444:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787829496.679445:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787829496.679446:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829496.679533:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787829496.679535:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829496.679537:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829496.679541:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d13eed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646835840:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829496.679545:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1787829496.679547:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1787829496.679548:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829496.679550:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1787829496.679553:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829496.679559:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787829496.679560:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839541349632 : -126234168201984 : ffff8d30d13eed00) 00000100:00000001:3.0:1787829496.679565:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1787829496.679577:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:3.0:1787829496.679582:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787829496.679585:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829496.679586:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829496.679587:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1787829496.679589:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829496.679594:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298969622 00000100:00000001:3.0:1787829496.679596:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298970880 00000100:00000001:3.0:1787829496.679598:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298969622 00000100:00000001:3.0:1787829496.679638:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298970880 00000100:00000001:3.0:1787829496.679640:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829496.679642:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829496.679658:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1787829496.679663:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829496/real 1787829496] req@ffff8d30d13eed00 x1874670646835840/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829521 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1787829496.679670:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787829496.679671:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829496.679674:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829496.679675:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829496.679677:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829496.679678:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829496.679680:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829496.679683:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d13eed00 x1874670646835840/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829521 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829496.679689:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829496.679691:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829496.679692:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829496.679694:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829496.679695:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829496.679697:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829496.679698:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829496.679700:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829496.679703:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829496.679704:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829496.679707:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d13eed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646835840:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829496.679710:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787829496.679712:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787829496.679713:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829496.679715:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787829496.679716:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829496.679718:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829496.679720:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d13ede00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646835968:192.168.203.143@tcp:8:kworker.0 00000100:00000001:3.0:1787829496.679723:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1787829496.679724:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1787829496.679726:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829496.679727:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1787829496.679731:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829496.679736:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787829496.679737:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839541345792 : -126234168205824 : ffff8d30d13ede00) 00000100:00000001:3.0:1787829496.679740:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1787829496.679753:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:3.0:1787829496.679757:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787829496.679759:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829496.679760:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829496.679762:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1787829496.679764:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829496.679767:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298969636 00000100:00000001:3.0:1787829496.679770:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298970880 00000100:00000001:3.0:1787829496.679772:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298969636 00000100:00000001:3.0:1787829496.679774:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298970880 00000100:00000001:3.0:1787829496.679775:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829496.679777:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829496.679778:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1787829496.679782:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829496/real 1787829496] req@ffff8d30d13ede00 x1874670646835968/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829521 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1787829496.679790:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787829496.679791:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829496.679794:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829496.679795:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829496.679796:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829496.679804:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829496.679806:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829496.679810:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d13ede00 x1874670646835968/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829521 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829496.679816:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829496.679819:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829496.679821:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829496.679823:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829496.679824:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829496.679825:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829496.679827:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829496.679830:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829496.679833:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829496.679835:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829496.679838:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d13ede00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646835968:192.168.203.143@tcp:8:kworker.0 00000100:00000001:3.0:1787829496.679841:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829496.679843:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829496.679844:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829496.679845:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829496.679848:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829496.679849:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829496.679850:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829496.679854:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829496.679855:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829496.679856:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829496.679857:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829496.679859:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829496.679860:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829496.679861:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829496.679863:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829496.679864:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829496.679865:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829496.679867:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829496.679868:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829496.679870:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829496.679871:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829496.679873:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829496.679874:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829496.679874:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829496.679876:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829496.679877:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829496.679878:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829496.679880:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829496.679880:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829496.679882:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829496.679883:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829496.679885:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829496.679889:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787829496.679890:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829496.679892:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1787829496.679893:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829497.574724:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829497.574728:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829497.574785:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829497.574788:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829497.574791:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829498.598681:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829498.598685:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829498.598719:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829498.598722:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829498.598724:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829499.622703:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829499.622710:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829499.622766:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829499.622770:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829499.622773:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829500.646659:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829500.646664:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829500.646708:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829500.646711:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829500.646714:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829501.670698:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829501.670705:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829501.670752:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829501.670755:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829501.670757:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787829501.798688:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829501.798695:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1787829501.798697:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787829501.798700:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787829501.798703:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787829501.798708:0:468:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 4302 00000100:00080000:3.0:1787829501.798715:0:468:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787829501.798717:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829501.798719:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:3.0:1787829501.798722:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1787829501.798724:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1787829501.798726:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:3.0:1787829501.798727:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:3.0:1787829501.798729:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829501.798730:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:3.0:1787829501.798733:0:468:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:3.0:1787829501.798735:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829501.798736:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787829501.798739:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829501.798741:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829501.798742:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:3.0:1787829501.798744:0:468:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829501.798752:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787829501.798753:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829501.798755:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787829501.798757:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787829501.798759:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829501.798767:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829501.798773:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c7016d00 x1874670646836096/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787829501.798780:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787829501.798782:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829501.798794:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787829501.798796:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829501.798798:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1787829501.798799:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787829501.798801:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787829501.798803:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787829501.798805:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4302 00000100:00080000:3.0:1787829501.798809:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1787829501.798812:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1787829501.798812:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829501.798816:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:1.0:1787829501.798817:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000020:00000001:3.0:1787829501.798819:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:1.0:1787829501.798820:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000020:00000001:3.0:1787829501.798822:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:1.0:1787829501.798823:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787829501.798826:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829501.798827:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 02000000:00000001:1.0:1787829501.798829:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000020:00000001:3.0:1787829501.798831:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 02000000:00000001:1.0:1787829501.798833:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829501.798837:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 02000000:00000001:1.0:1787829501.798838:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829501.798842:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00100000:1.0:1787829501.798844:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c7016d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646836096:192.168.203.143@tcp:250:kworker.0 00000100:00000001:1.0:1787829501.798850:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00080000:3.0:1787829501.798850:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 02000000:00000001:1.0:1787829501.798854:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:3.0:1787829501.798854:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829501.798856:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1787829501.798857:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829501.798860:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1787829501.798861:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829501.798863:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829501.798864:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829501.798868:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:3.0:1787829501.798871:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1787829501.798874:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829501.798875:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829501.798877:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 02000000:00000001:3.0:1787829501.798879:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:1.0:1787829501.798880:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369547008 : -126234340004608 : ffff8d30c7016d00) 02000000:00000001:3.0:1787829501.798882:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829501.798885:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:3.0:1787829501.798885:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787829501.798889:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787829501.798890:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829501.798895:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829501.798899:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c70170c0 x1874670646836224/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787829501.798904:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787829501.798905:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829501.798908:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787829501.798910:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829501.798912:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1787829501.798913:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787829501.798915:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:1.0:1787829501.798916:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:3.0:1787829501.798918:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787829501.798922:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4302 00000100:00000001:1.0:1787829501.798925:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00080000:3.0:1787829501.798926:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787829501.798929:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829501.798930:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:1.0:1787829501.798931:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829501.798935:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829501.798935:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1787829501.798937:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:1.0:1787829501.798938:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000020:00000001:3.0:1787829501.798939:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000020:00000001:3.0:1787829501.798942:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:3.0:1787829501.798944:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787829501.798945:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829501.798945:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:1.0:1787829501.798951:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4298976000 00000100:00080000:3.0:1787829501.798951:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:3.0:1787829501.798957:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829501.798958:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787829501.798959:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829501.798961:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829501.798962:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 00000100:00000001:3.0:1787829501.798965:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:3.0:1787829501.798967:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00000001:1.0:1787829501.798968:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4298976000 00000008:00000001:3.0:1787829501.798969:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829501.798974:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829501.798975:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 02000000:00000001:3.0:1787829501.798978:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:1.0:1787829501.798979:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829501.798980:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829501.798983:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829501.798984:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787829501.798986:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:1.0:1787829501.798987:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 02000000:00000001:3.0:1787829501.798988:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829501.798993:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829501.798994:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829501/real 1787829501] req@ffff8d30c7016d00 x1874670646836096/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829526 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787829501.798998:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c7017840 x1874670646836352/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000001:1.0:1787829501.799004:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787829501.799005:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787829501.799007:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787829501.799009:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787829501.799010:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829501.799014:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829501.799016:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829501.799021:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00100000:1.0:1787829501.799023:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c7016d00 x1874670646836096/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829526 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829501.799024:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829501.799028:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787829501.799032:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1787829501.799037:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:3.0:1787829501.799037:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1787829501.799040:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:1.0:1787829501.799041:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829501.799042:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787829501.799046:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829501.799047:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:1.0:1787829501.799049:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829501.799051:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:3.0:1787829501.799052:0:468:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4302 00000100:00000001:1.0:1787829501.799053:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787829501.799056:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787829501.799059:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829501.799060:0:468:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1787829501.799062:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787829501.799065:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000001:3.0:1787829501.799065:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829501.799068:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00100000:1.0:1787829501.799070:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c7016d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646836096:192.168.203.143@tcp:250:kworker.0 00000020:00000001:3.0:1787829501.799075:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:1.0:1787829501.799076:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829501.799079:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000020:00000001:3.0:1787829501.799079:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:1.0:1787829501.799082:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 00000020:00000001:3.0:1787829501.799083:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 02000000:00000001:1.0:1787829501.799084:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787829501.799087:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829501.799088:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000020:00000001:1.0:1787829501.799090:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000100:00000001:3.0:1787829501.799092:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000020:00000001:1.0:1787829501.799093:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 00000100:00000001:3.0:1787829501.799096:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 02000000:00000001:1.0:1787829501.799099:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 00000100:00080000:3.0:1787829501.799102:0:468:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 02000000:00000001:1.0:1787829501.799103:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829501.799105:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829501.799108:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 02000000:00000001:3.0:1787829501.799109:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1787829501.799112:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:3.0:1787829501.799112:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829501.799115:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829501.799117:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829501.799118:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:3.0:1787829501.799120:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1787829501.799123:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829501.799124:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829501.799126:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 02000000:00000001:3.0:1787829501.799127:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:1.0:1787829501.799130:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829501.799130:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829501.799135:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:1.0:1787829501.799136:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:3.0:1787829501.799137:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:1.0:1787829501.799141:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 02000000:00000001:3.0:1787829501.799141:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829501.799144:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787829501.799146:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787829501.799147:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829501.799148:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829501.799152:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000100:00100000:3.0:1787829501.799154:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c7014000 x1874670646836480/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:1.0:1787829501.799155:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829501.799160:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829501.799167:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c70170c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646836224:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829501.799168:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787829501.799172:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:3.0:1787829501.799172:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829501.799176:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:3.0:1787829501.799177:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829501.799178:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829501.799181:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787829501.799184:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829501.799189:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787829501.799190:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369547968 : -126234340003648 : ffff8d30c70170c0) 00000100:00000001:1.0:1787829501.799193:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1787829501.799207:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787829501.799210:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787829501.799212:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829501.799213:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829501.799214:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787829501.799215:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829501.799218:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298974742 00000100:00000001:1.0:1787829501.799220:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298976000 00000100:00000001:1.0:1787829501.799222:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298974742 00000100:00000001:1.0:1787829501.799223:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298976000 00000100:00000001:1.0:1787829501.799224:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829501.799225:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829501.799226:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787829501.799229:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829501/real 1787829501] req@ffff8d30c70170c0 x1874670646836224/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829526 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787829501.799235:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787829501.799235:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787829501.799237:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787829501.799238:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787829501.799238:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829501.799239:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829501.799240:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787829501.799242:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c70170c0 x1874670646836224/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829526 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787829501.799246:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829501.799248:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787829501.799249:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829501.799250:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829501.799250:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829501.799251:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787829501.799252:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787829501.799254:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829501.799255:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787829501.799256:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787829501.799258:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c70170c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646836224:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787829501.799260:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787829501.799261:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787829501.799262:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829501.799263:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787829501.799264:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829501.799265:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829501.799266:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c7017840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646836352:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787829501.799268:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787829501.799269:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787829501.799270:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829501.799271:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787829501.799273:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829501.799275:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787829501.799276:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369549888 : -126234340001728 : ffff8d30c7017840) 00000100:00000001:1.0:1787829501.799278:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1787829501.799283:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787829501.799285:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787829501.799286:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829501.799287:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829501.799288:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787829501.799290:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829501.799291:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298974742 00000100:00000001:1.0:1787829501.799293:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298976000 00000100:00000001:1.0:1787829501.799294:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298974742 00000100:00000001:1.0:1787829501.799295:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298976000 00000100:00000001:1.0:1787829501.799297:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829501.799298:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829501.799298:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787829501.799300:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829501/real 1787829501] req@ffff8d30c7017840 x1874670646836352/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829526 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787829501.799305:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787829501.799305:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787829501.799307:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787829501.799308:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787829501.799309:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829501.799311:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829501.799312:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787829501.799315:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c7017840 x1874670646836352/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829526 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787829501.799321:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829501.799323:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787829501.799325:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829501.799327:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829501.799327:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829501.799329:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787829501.799330:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787829501.799332:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829501.799335:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787829501.799337:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787829501.799339:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c7017840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646836352:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787829501.799342:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829501.799343:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829501.799344:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787829501.799346:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787829501.799348:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787829501.799349:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787829501.799350:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787829501.799352:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787829501.799354:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787829501.799355:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787829501.799356:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829501.799358:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829501.799358:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787829501.799360:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787829501.799361:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787829501.799362:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787829501.799363:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787829501.799365:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787829501.799366:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787829501.799367:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787829501.799368:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829501.799370:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829501.799372:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787829501.799373:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829501.799375:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787829501.799376:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1787829501.799377:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787829501.799378:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787829501.799380:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829501.799382:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787829501.799383:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829501.799384:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829501.799387:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c7014000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646836480:192.168.203.143@tcp:8:kworker.0 00000100:00000001:1.0:1787829501.799390:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787829501.799391:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787829501.799392:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829501.799394:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787829501.799396:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829501.799400:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787829501.799402:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369535488 : -126234340016128 : ffff8d30c7014000) 00000100:00000001:1.0:1787829501.799404:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1787829501.799414:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787829501.799418:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787829501.799421:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829501.799422:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829501.799423:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787829501.799425:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829501.799427:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298974755 00000100:00000001:1.0:1787829501.799429:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298976000 00000100:00000001:1.0:1787829501.799431:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298974755 00000100:00000001:1.0:1787829501.799434:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298976000 00000100:00000001:1.0:1787829501.799435:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829501.799437:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829501.799439:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787829501.799442:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829501/real 1787829501] req@ffff8d30c7014000 x1874670646836480/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829526 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787829501.799448:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787829501.799449:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787829501.799451:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787829501.799452:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787829501.799453:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829501.799455:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829501.799456:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787829501.799459:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c7014000 x1874670646836480/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829526 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787829501.799465:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829501.799467:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787829501.799468:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829501.799469:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829501.799470:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829501.799472:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787829501.799474:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787829501.799475:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829501.799478:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787829501.799480:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787829501.799482:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c7014000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646836480:192.168.203.143@tcp:8:kworker.0 00000100:00000001:1.0:1787829501.799485:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829501.799487:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829501.799488:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787829501.799489:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787829501.799491:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787829501.799492:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787829501.799493:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787829501.799494:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787829501.799495:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787829501.799497:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787829501.799498:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829501.799499:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829501.799502:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787829501.799503:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829501.799505:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787829501.799506:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829502.694678:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829502.694683:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829502.694735:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829502.694738:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829502.694742:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829503.718685:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829503.718689:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829503.718732:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829503.718735:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829503.718738:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829504.742660:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829504.742666:0:815:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1787829504.742667:0:815:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1787829504.742669:0:815:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829504.742672:0:815:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1787829504.742674:0:815:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829504.742675:0:815:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829504.742677:0:815:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829504.742724:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829504.742728:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829504.742730:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829505.766704:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829505.766710:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829505.766792:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829505.766797:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829505.766801:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829506.790675:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829506.790680:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829506.790731:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829506.790734:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829506.790736:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787829506.918682:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829506.918689:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1787829506.918691:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787829506.918694:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787829506.918698:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787829506.918709:0:468:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 4307 00000100:00080000:3.0:1787829506.918716:0:468:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787829506.918718:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829506.918720:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:3.0:1787829506.918722:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1787829506.918724:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1787829506.918727:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:3.0:1787829506.918728:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:3.0:1787829506.918730:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829506.918731:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:3.0:1787829506.918734:0:468:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:3.0:1787829506.918736:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829506.918737:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787829506.918740:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829506.918742:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829506.918743:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:3.0:1787829506.918745:0:468:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829506.918752:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787829506.918753:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829506.918756:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787829506.918758:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787829506.918761:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829506.918769:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829506.918776:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c7016940 x1874670646836608/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787829506.918787:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787829506.918790:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829506.918802:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787829506.918804:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829506.918807:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1787829506.918808:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787829506.918809:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787829506.918810:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787829506.918812:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4307 00000100:00080000:3.0:1787829506.918814:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787829506.918815:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829506.918816:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:3.0:1787829506.918818:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1787829506.918819:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1787829506.918820:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000020:00000001:3.0:1787829506.918822:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:3.0:1787829506.918824:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829506.918825:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:3.0:1787829506.918827:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:3.0:1787829506.918829:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829506.918830:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:3.0:1787829506.918830:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:0.0:1787829506.918836:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 02000000:00000001:3.0:1787829506.918836:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787829506.918838:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 02000000:00000001:3.0:1787829506.918839:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829506.918841:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787829506.918842:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1787829506.918844:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000008:00100000:3.0:1787829506.918846:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 02000000:00000001:0.0:1787829506.918849:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000008:00000001:3.0:1787829506.918849:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829506.918853:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829506.918854:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787829506.918858:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829506.918859:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787829506.918862:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787829506.918863:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c7016940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646836608:192.168.203.143@tcp:250:kworker.0 00000100:00000001:3.0:1787829506.918864:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787829506.918867:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:0.0:1787829506.918868:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1787829506.918871:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829506.918872:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787829506.918875:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829506.918878:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:3.0:1787829506.918880:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829506.918882:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829506.918888:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c7017c00 x1874670646836736/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787829506.918893:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787829506.918896:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369546048 : -126234340005568 : ffff8d30c7016940) 00000100:00000001:3.0:1787829506.918899:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787829506.918901:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:3.0:1787829506.918904:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829506.918909:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787829506.918912:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829506.918916:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1787829506.918917:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787829506.918919:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787829506.918929:0:468:0:(import.c:485:import_select_connection()) Process entered 00000800:00000100:0.0:1787829506.918931:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:0.0:1787829506.918940:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1787829506.918943:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787829506.918945:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829506.918946:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787829506.918951:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829506.918954:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4298981120 00000100:00000001:0.0:1787829506.918960:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1787829506.918965:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4298981120 00000100:00000001:0.0:1787829506.918968:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1787829506.918969:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787829506.918971:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829506.918972:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787829506.918976:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829506/real 1787829506] req@ffff8d30c7016940 x1874670646836608/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829531 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787829506.918986:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787829506.918988:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787829506.918990:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787829506.918993:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787829506.918994:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829506.918996:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787829506.918998:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787829506.919008:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c7016940 x1874670646836608/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829531 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787829506.919016:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829506.919020:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829506.919022:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4307 00000100:00000001:0.0:1787829506.919024:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829506.919028:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787829506.919028:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:3.0:1787829506.919030:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787829506.919031:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787829506.919033:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829506.919035:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00080000:0.0:1787829506.919036:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00000001:3.0:1787829506.919038:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:0.0:1787829506.919041:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787829506.919044:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000020:00000001:3.0:1787829506.919044:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1787829506.919047:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00100000:0.0:1787829506.919049:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c7016940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646836608:192.168.203.143@tcp:250:kworker.0 00000020:00000001:3.0:1787829506.919051:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:0.0:1787829506.919053:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829506.919055:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787829506.919056:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787829506.919058:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787829506.919060:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787829506.919063:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829506.919063:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000020:00000001:0.0:1787829506.919065:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 00000100:00000001:3.0:1787829506.919069:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 02000000:00000001:0.0:1787829506.919070:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787829506.919073:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829506.919073:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:0.0:1787829506.919075:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787829506.919078:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829506.919080:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829506.919085:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787829506.919086:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829506.919088:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787829506.919089:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787829506.919089:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1787829506.919098:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787829506.919099:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829506.919100:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787829506.919101:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787829506.919103:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787829506.919105:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c7017c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646836736:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787829506.919108:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787829506.919109:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787829506.919110:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829506.919111:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1787829506.919113:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829506.919118:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787829506.919119:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369550848 : -126234340000768 : ffff8d30c7017c00) 00000100:00000001:0.0:1787829506.919122:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1787829506.919130:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:0.0:1787829506.919133:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1787829506.919135:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787829506.919136:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829506.919137:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787829506.919140:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829506.919142:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298979862 00000100:00000001:0.0:1787829506.919144:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298981120 00000100:00000001:0.0:1787829506.919146:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298979862 00000100:00000001:0.0:1787829506.919148:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298981120 00000100:00000001:0.0:1787829506.919149:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787829506.919150:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829506.919151:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787829506.919154:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829506/real 1787829506] req@ffff8d30c7017c00 x1874670646836736/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829531 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787829506.919158:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787829506.919159:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787829506.919162:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787829506.919163:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787829506.919164:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829506.919165:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787829506.919166:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787829506.919169:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c7017c00 x1874670646836736/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829531 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787829506.919174:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787829506.919175:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829506.919179:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829506.919180:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829506.919181:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829506.919185:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 02000000:00000001:3.0:1787829506.919186:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:0.0:1787829506.919187:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829506.919191:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 02000000:00000001:3.0:1787829506.919192:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787829506.919193:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787829506.919198:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829506.919201:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787829506.919203:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787829506.919205:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c7017c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646836736:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787829506.919207:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829506.919208:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787829506.919208:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787829506.919209:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787829506.919211:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787829506.919212:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787829506.919213:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787829506.919215:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787829506.919215:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787829506.919217:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787829506.919218:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829506.919219:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829506.919221:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787829506.919222:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829506.919224:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787829506.919226:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829506.919238:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829506.919241:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:3.0:1787829506.919244:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1787829506.919248:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829506.919249:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829506.919254:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787829506.919256:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829506.919258:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787829506.919260:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787829506.919262:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829506.919266:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829506.919271:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c7017c00 x1874670646836864/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787829506.919279:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787829506.919281:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829506.919295:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787829506.919297:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829506.919299:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1787829506.919301:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787829506.919302:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787829506.919304:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787829506.919306:0:468:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4307 00000100:00000001:0.0:1787829506.919309:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00080000:3.0:1787829506.919310:0:468:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787829506.919313:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787829506.919313:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829506.919317:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787829506.919317:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:0.0:1787829506.919320:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000020:00000001:3.0:1787829506.919321:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:0.0:1787829506.919323:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829506.919324:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1787829506.919327:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 02000000:00000001:0.0:1787829506.919328:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000020:00000001:3.0:1787829506.919329:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 02000000:00000001:0.0:1787829506.919330:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829506.919332:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 02000000:00000001:0.0:1787829506.919333:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829506.919334:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00100000:0.0:1787829506.919339:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c7017c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646836864:192.168.203.143@tcp:38:kworker.0 00000100:00080000:3.0:1787829506.919339:0:468:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787829506.919345:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:3.0:1787829506.919345:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829506.919348:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787829506.919349:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1787829506.919351:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787829506.919352:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829506.919356:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1787829506.919357:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829506.919359:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 02000000:00000001:0.0:1787829506.919360:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000008:00100000:3.0:1787829506.919361:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1787829506.919364:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829506.919365:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829506.919366:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787829506.919369:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369550848 : -126234340000768 : ffff8d30c7017c00) 02000000:00000001:3.0:1787829506.919369:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787829506.919372:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829506.919374:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:3.0:1787829506.919374:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787829506.919377:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787829506.919378:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829506.919382:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829506.919385:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c7016940 x1874670646836992/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787829506.919390:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:0.0:1787829506.919395:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787829506.919396:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787829506.919399:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829506.919399:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829506.919402:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829506.919403:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829506.919405:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787829506.919408:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829506.919411:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298979862 00000100:00000001:0.0:1787829506.919413:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298981120 00000100:00000001:0.0:1787829506.919414:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298979862 00000100:00000001:0.0:1787829506.919416:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298981120 00000100:00000001:0.0:1787829506.919418:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787829506.919419:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829506.919420:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787829506.919423:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829506/real 1787829506] req@ffff8d30c7017c00 x1874670646836864/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829531 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787829506.919428:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787829506.919428:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787829506.919430:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787829506.919431:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787829506.919432:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829506.919433:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787829506.919434:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787829506.919436:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c7017c00 x1874670646836864/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829531 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787829506.919440:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829506.919441:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787829506.919442:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829506.919443:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787829506.919444:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829506.919445:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787829506.919445:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787829506.919447:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829506.919448:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787829506.919450:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787829506.919451:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c7017c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646836864:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787829506.919453:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829506.919454:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787829506.919455:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787829506.919456:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787829506.919457:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787829506.919458:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787829506.919459:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787829506.919460:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787829506.919461:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787829506.919462:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787829506.919463:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829506.919464:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829506.919466:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787829506.919467:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829506.919468:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787829506.919469:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787829506.919470:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1787829506.919471:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787829506.919472:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829506.919473:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787829506.919474:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787829506.919475:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787829506.919476:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c7016940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646836992:192.168.203.143@tcp:8:kworker.0 00000100:00000001:0.0:1787829506.919478:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787829506.919479:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787829506.919480:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829506.919481:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1787829506.919482:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829506.919485:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787829506.919486:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369546048 : -126234340005568 : ffff8d30c7016940) 00000100:00000001:0.0:1787829506.919488:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1787829506.919494:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:0.0:1787829506.919496:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1787829506.919497:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787829506.919498:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829506.919499:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787829506.919500:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829506.919502:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298979875 00000100:00000001:0.0:1787829506.919503:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298981120 00000100:00000001:0.0:1787829506.919505:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298979875 00000100:00000001:0.0:1787829506.919506:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298981120 00000100:00000001:0.0:1787829506.919507:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787829506.919508:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829506.919509:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787829506.919511:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829506/real 1787829506] req@ffff8d30c7016940 x1874670646836992/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829531 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787829506.919515:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787829506.919516:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787829506.919517:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787829506.919518:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787829506.919519:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829506.919520:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787829506.919521:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787829506.919523:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c7016940 x1874670646836992/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829531 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787829506.919527:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829506.919528:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787829506.919529:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829506.919530:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787829506.919531:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829506.919531:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787829506.919532:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787829506.919534:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829506.919535:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787829506.919536:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787829506.919538:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c7016940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646836992:192.168.203.143@tcp:8:kworker.0 00000100:00000001:0.0:1787829506.919540:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829506.919541:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787829506.919541:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787829506.919542:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787829506.919543:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787829506.919544:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787829506.919545:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787829506.919546:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787829506.919547:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787829506.919547:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787829506.919548:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829506.919549:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829506.919550:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787829506.919551:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829506.919552:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787829506.919553:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829507.814687:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829507.814692:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829507.814741:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829507.814744:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829507.814747:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829508.838681:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829508.838687:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829508.838738:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829508.838742:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829508.838745:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829509.862677:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829509.862682:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829509.862729:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829509.862732:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829509.862734:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829510.886680:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829510.886685:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829510.886715:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829510.886719:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829510.886722:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829511.910682:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829511.910689:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829511.910747:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829511.910750:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829511.910754:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787829512.038682:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829512.038689:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1787829512.038691:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787829512.038699:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787829512.038705:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787829512.038710:0:468:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 4312 00000100:00080000:3.0:1787829512.038717:0:468:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787829512.038719:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829512.038720:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:3.0:1787829512.038723:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1787829512.038725:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1787829512.038728:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:3.0:1787829512.038730:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:3.0:1787829512.038733:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829512.038734:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:3.0:1787829512.038738:0:468:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:3.0:1787829512.038740:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829512.038743:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787829512.038746:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829512.038749:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829512.038751:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:3.0:1787829512.038753:0:468:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829512.038763:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787829512.038765:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829512.038768:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787829512.038771:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787829512.038775:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829512.038785:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829512.038793:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c7014000 x1874670646837120/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787829512.038803:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787829512.038807:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829512.038821:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787829512.038826:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829512.038830:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1787829512.038831:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787829512.038832:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787829512.038835:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787829512.038837:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4312 00000100:00080000:3.0:1787829512.038842:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787829512.038844:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829512.038845:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:3.0:1787829512.038848:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1787829512.038849:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1787829512.038851:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:0.0:1787829512.038853:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:3.0:1787829512.038854:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:0.0:1787829512.038858:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787829512.038858:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829512.038862:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787829512.038862:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:0.0:1787829512.038866:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00080000:3.0:1787829512.038868:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787829512.038869:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829512.038872:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829512.038875:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787829512.038875:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787829512.038880:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829512.038882:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787829512.038885:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829512.038886:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829512.038888:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:3.0:1787829512.038892:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1787829512.038896:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829512.038898:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787829512.038901:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c7014000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646837120:192.168.203.143@tcp:250:kworker.0 02000000:00000001:3.0:1787829512.038902:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787829512.038904:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829512.038907:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787829512.038909:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:0.0:1787829512.038911:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1787829512.038911:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829512.038917:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:3.0:1787829512.038918:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829512.038923:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829512.038925:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c7017840 x1874670646837248/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:0.0:1787829512.038927:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:3.0:1787829512.038932:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 02000000:00000001:0.0:1787829512.038933:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829512.038934:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829512.038938:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787829512.038941:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829512.038944:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1787829512.038946:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787829512.038947:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787829512.038950:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787829512.038952:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4312 00000100:00080000:3.0:1787829512.038956:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787829512.038958:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829512.038959:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787829512.038961:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:0.0:1787829512.038963:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369535488 : -126234340016128 : ffff8d30c7014000) 00000020:00000001:3.0:1787829512.038965:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1787829512.038968:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:0.0:1787829512.038970:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000020:00000001:3.0:1787829512.038971:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000020:00000001:3.0:1787829512.038976:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:3.0:1787829512.038978:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829512.038980:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:3.0:1787829512.038983:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:3.0:1787829512.038986:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829512.038988:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787829512.038991:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829512.038993:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829512.038994:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:3.0:1787829512.038997:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1787829512.038999:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829512.039001:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829512.039005:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000800:00000100:0.0:1787829512.039006:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 02000000:00000001:3.0:1787829512.039008:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829512.039011:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787829512.039013:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787829512.039015:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829512.039016:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787829512.039020:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829512.039022:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787829512.039025:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829512.039028:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00100000:3.0:1787829512.039029:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c70170c0 x1874670646837376/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787829512.039032:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829512.039038:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4298986240 00000100:00000001:3.0:1787829512.039042:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787829512.039045:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829512.039048:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 00000100:00000001:3.0:1787829512.039048:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787829512.039053:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1787829512.039055:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4298986240 00000100:00080000:3.0:1787829512.039058:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1787829512.039060:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 00000100:00000001:3.0:1787829512.039060:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787829512.039064:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787829512.039066:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829512.039067:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:0.0:1787829512.039071:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787829512.039072:0:468:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4312 00000100:00000001:0.0:1787829512.039076:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00080000:3.0:1787829512.039077:0:468:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787829512.039081:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829512.039082:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00100000:0.0:1787829512.039085:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829512/real 1787829512] req@ffff8d30c7014000 x1874670646837120/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829537 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:3.0:1787829512.039086:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1787829512.039088:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1787829512.039091:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000020:00000001:3.0:1787829512.039095:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:3.0:1787829512.039100:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829512.039103:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000400:00000001:0.0:1787829512.039107:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787829512.039110:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787829512.039113:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787829512.039115:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00080000:3.0:1787829512.039116:0:468:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787829512.039117:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829512.039121:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829512.039123:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829512.039124:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:3.0:1787829512.039129:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00100000:0.0:1787829512.039132:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c7014000 x1874670646837120/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829537 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1787829512.039134:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829512.039139:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829512.039143:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1787829512.039144:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000008:00100000:3.0:1787829512.039146:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00000001:0.0:1787829512.039148:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000008:00000001:3.0:1787829512.039151:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829512.039152:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829512.039155:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829512.039155:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829512.039157:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829512.039160:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787829512.039162:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 02000000:00000001:3.0:1787829512.039162:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00080000:0.0:1787829512.039166:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 02000000:00000001:3.0:1787829512.039166:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829512.039171:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00080000:0.0:1787829512.039172:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 02000000:00000001:3.0:1787829512.039175:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:0.0:1787829512.039176:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 02000000:00000001:3.0:1787829512.039179:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787829512.039180:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c7014000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646837120:192.168.203.143@tcp:250:kworker.0 00000100:00000001:0.0:1787829512.039187:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829512.039187:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829512.039190:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787829512.039192:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787829512.039194:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787829512.039197:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00100000:3.0:1787829512.039197:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c7016d00 x1874670646837504/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000020:00000001:0.0:1787829512.039200:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787829512.039203:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787829512.039207:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787829512.039209:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829512.039210:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787829512.039211:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787829512.039214:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829512.039215:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829512.039216:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829512.039219:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829512.039223:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787829512.039225:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829512.039228:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787829512.039229:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787829512.039230:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1787829512.039231:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787829512.039232:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829512.039234:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787829512.039236:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787829512.039238:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787829512.039241:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c7017840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646837248:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787829512.039244:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787829512.039245:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787829512.039246:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829512.039248:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1787829512.039250:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829512.039255:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787829512.039257:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369549888 : -126234340001728 : ffff8d30c7017840) 00000100:00000001:0.0:1787829512.039260:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1787829512.039272:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:0.0:1787829512.039277:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1787829512.039279:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787829512.039280:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829512.039282:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787829512.039285:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829512.039289:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298984982 00000100:00000001:0.0:1787829512.039292:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298986240 00000100:00000001:0.0:1787829512.039294:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298984982 00000100:00000001:0.0:1787829512.039296:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298986240 00000100:00000001:0.0:1787829512.039297:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787829512.039299:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829512.039300:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787829512.039303:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829512/real 1787829512] req@ffff8d30c7017840 x1874670646837248/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829537 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787829512.039309:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787829512.039309:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787829512.039311:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787829512.039312:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787829512.039313:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829512.039314:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787829512.039315:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787829512.039317:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c7017840 x1874670646837248/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829537 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787829512.039321:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829512.039323:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787829512.039324:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829512.039325:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787829512.039325:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829512.039326:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787829512.039327:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787829512.039329:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829512.039331:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787829512.039332:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787829512.039335:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c7017840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646837248:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787829512.039339:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1787829512.039340:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787829512.039341:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829512.039343:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787829512.039345:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787829512.039347:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787829512.039350:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c70170c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646837376:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787829512.039353:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787829512.039355:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787829512.039356:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829512.039358:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1787829512.039360:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829512.039365:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787829512.039366:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369547968 : -126234340003648 : ffff8d30c70170c0) 00000100:00000001:0.0:1787829512.039370:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1787829512.039380:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:0.0:1787829512.039382:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1787829512.039384:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787829512.039385:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829512.039386:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787829512.039388:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829512.039390:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298984982 00000100:00000001:0.0:1787829512.039392:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298986240 00000100:00000001:0.0:1787829512.039393:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298984982 00000100:00000001:0.0:1787829512.039394:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298986240 00000100:00000001:0.0:1787829512.039395:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787829512.039396:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829512.039397:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787829512.039400:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829512/real 1787829512] req@ffff8d30c70170c0 x1874670646837376/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829537 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787829512.039404:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787829512.039405:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787829512.039406:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787829512.039407:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787829512.039407:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829512.039408:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787829512.039409:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787829512.039411:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c70170c0 x1874670646837376/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829537 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787829512.039415:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829512.039416:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787829512.039418:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829512.039419:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787829512.039419:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829512.039420:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787829512.039421:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787829512.039423:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829512.039424:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787829512.039425:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787829512.039427:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c70170c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646837376:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787829512.039430:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1787829512.039430:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787829512.039431:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829512.039432:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787829512.039433:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787829512.039434:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787829512.039436:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c7016d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646837504:192.168.203.143@tcp:8:kworker.0 00000100:00000001:0.0:1787829512.039438:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787829512.039439:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787829512.039440:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829512.039441:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1787829512.039442:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829512.039445:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787829512.039446:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369547008 : -126234340004608 : ffff8d30c7016d00) 00000100:00000001:0.0:1787829512.039448:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1787829512.039453:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:0.0:1787829512.039455:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1787829512.039456:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787829512.039457:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829512.039458:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787829512.039460:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829512.039461:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298984995 00000100:00000001:0.0:1787829512.039463:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298986240 00000100:00000001:0.0:1787829512.039464:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298984995 00000100:00000001:0.0:1787829512.039465:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298986240 00000100:00000001:0.0:1787829512.039466:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787829512.039467:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829512.039468:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787829512.039470:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829512/real 1787829512] req@ffff8d30c7016d00 x1874670646837504/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829537 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787829512.039479:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787829512.039479:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787829512.039480:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787829512.039481:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787829512.039482:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829512.039483:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787829512.039484:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787829512.039485:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c7016d00 x1874670646837504/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829537 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787829512.039489:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829512.039490:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787829512.039491:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829512.039492:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787829512.039492:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829512.039493:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787829512.039494:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787829512.039495:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829512.039497:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787829512.039499:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787829512.039500:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c7016d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646837504:192.168.203.143@tcp:8:kworker.0 00000100:00000001:0.0:1787829512.039502:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829512.039504:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787829512.039504:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787829512.039505:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787829512.039506:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787829512.039507:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787829512.039508:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787829512.039509:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787829512.039510:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787829512.039511:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787829512.039511:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829512.039513:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787829512.039513:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787829512.039514:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787829512.039515:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787829512.039516:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787829512.039516:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787829512.039517:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787829512.039518:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787829512.039519:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787829512.039519:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829512.039520:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787829512.039521:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787829512.039521:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787829512.039522:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787829512.039523:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787829512.039523:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787829512.039524:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787829512.039524:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787829512.039525:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787829512.039526:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829512.039527:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829512.039529:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787829512.039530:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829512.039531:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787829512.039531:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829512.934679:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829512.934685:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829512.934739:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829512.934743:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829512.934746:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829513.958696:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829513.958701:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829513.958750:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829513.958753:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829513.958755:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829514.982688:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829514.982694:0:815:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1787829514.982695:0:815:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1787829514.982697:0:815:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829514.982701:0:815:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1787829514.982702:0:815:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829514.982704:0:815:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829514.982706:0:815:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829514.982753:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829514.982756:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829514.982758:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829516.006680:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829516.006685:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829516.006744:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829516.006748:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829516.006751:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829517.030707:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829517.030714:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829517.030778:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829517.030782:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829517.030785:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787829517.158757:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829517.158766:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1787829517.158769:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787829517.158772:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787829517.158778:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787829517.158785:0:40798:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 4317 00000100:00080000:0.0:1787829517.158796:0:40798:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787829517.158798:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829517.158800:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787829517.158804:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787829517.158806:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787829517.158809:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:0.0:1787829517.158811:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:0.0:1787829517.158813:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829517.158814:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:0.0:1787829517.158818:0:40798:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787829517.158820:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829517.158822:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787829517.158826:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787829517.158828:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829517.158830:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1787829517.158831:0:40798:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829517.158845:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787829517.158846:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829517.158850:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787829517.158852:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787829517.158857:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829517.158871:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787829517.158880:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d26940 x1874670646837632/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787829517.158889:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787829517.158893:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829517.158911:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787829517.158914:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829517.158918:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1787829517.158919:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787829517.158921:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787829517.158924:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787829517.158927:0:40798:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4317 00000100:00080000:0.0:1787829517.158931:0:40798:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787829517.158934:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829517.158935:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787829517.158938:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787829517.158940:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787829517.158943:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000020:00000001:0.0:1787829517.158946:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:0.0:1787829517.158948:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829517.158950:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:0.0:1787829517.158954:0:40798:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787829517.158957:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829517.158960:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787829517.158967:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787829517.158969:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829517.158971:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:1.0:1787829517.158976:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000008:00100000:0.0:1787829517.158977:0:40798:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1787829517.158984:0:40798:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829517.158989:0:40798:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829517.158989:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1787829517.158996:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 02000000:00000001:0.0:1787829517.158998:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787829517.159000:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829517.159001:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787829517.159002:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787829517.159007:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:1.0:1787829517.159008:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829517.159010:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829517.159022:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829517.159024:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787829517.159029:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00100000:0.0:1787829517.159030:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d27c00 x1874670646837760/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:1.0:1787829517.159042:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829517.159053:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787829517.159055:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829517.159058:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829517.159061:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d26940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646837632:192.168.203.143@tcp:250:kworker.0 00000100:00080000:0.0:1787829517.159062:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829517.159074:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1787829517.159077:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:1.0:1787829517.159083:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00080000:0.0:1787829517.159085:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787829517.159089:0:40798:0:(import.c:485:import_select_connection()) Process entered 02000000:00000001:1.0:1787829517.159091:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000100:00080000:0.0:1787829517.159092:0:40798:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4317 00000100:00080000:0.0:1787829517.159098:0:40798:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 02000000:00000001:1.0:1787829517.159098:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829517.159102:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829517.159107:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 02000000:00000001:1.0:1787829517.159109:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000020:00000001:0.0:1787829517.159113:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787829517.159116:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787829517.159119:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 02000000:00000001:1.0:1787829517.159120:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787829517.159126:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:0.0:1787829517.159132:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829517.159133:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:0.0:1787829517.159139:0:40798:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:1.0:1787829517.159141:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787829517.159143:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829517.159145:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601346880 : -126234108204736 : ffff8d30d4d26940) 02000000:00000001:0.0:1787829517.159147:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787829517.159150:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829517.159153:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 02000000:00000001:0.0:1787829517.159154:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829517.159158:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1787829517.159160:0:40798:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1787829517.159163:0:40798:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829517.159165:0:40798:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829517.159168:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787829517.159169:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829517.159171:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787829517.159172:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787829517.159173:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829517.159177:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787829517.159181:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d25680 x1874670646837888/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787829517.159187:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787829517.159189:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:1.0:1787829517.159197:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:0.0:1787829517.159200:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787829517.159204:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829517.159207:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1787829517.159208:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787829517.159209:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787829517.159212:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:1.0:1787829517.159212:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00080000:0.0:1787829517.159217:0:40798:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4317 00000100:00000001:1.0:1787829517.159221:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00080000:0.0:1787829517.159222:0:40798:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787829517.159227:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787829517.159229:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829517.159230:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787829517.159234:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787829517.159240:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:1.0:1787829517.159241:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000020:00000001:0.0:1787829517.159244:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000020:00000001:0.0:1787829517.159248:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:0.0:1787829517.159250:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829517.159251:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:1.0:1787829517.159252:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787829517.159256:0:40798:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787829517.159259:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829517.159261:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1787829517.159261:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4298991360 02000000:00000001:0.0:1787829517.159263:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787829517.159269:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829517.159271:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1787829517.159272:0:40798:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1787829517.159274:0:40798:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829517.159275:0:40798:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829517.159276:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 02000000:00000001:0.0:1787829517.159279:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787829517.159281:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829517.159283:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787829517.159285:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787829517.159287:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829517.159287:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4298991360 00000100:00000001:0.0:1787829517.159291:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787829517.159297:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d252c0 x1874670646838016/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787829517.159298:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 00000100:00000001:1.0:1787829517.159307:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787829517.159309:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787829517.159313:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829517.159321:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829517.159323:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829517.159334:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787829517.159344:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829517/real 1787829517] req@ffff8d30d4d26940 x1874670646837632/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829542 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787829517.159355:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787829517.159357:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787829517.159361:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787829517.159364:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787829517.159365:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829517.159368:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829517.159369:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787829517.159374:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d26940 x1874670646837632/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829542 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787829517.159381:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829517.159384:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787829517.159387:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829517.159388:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829517.159389:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829517.159392:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787829517.159393:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787829517.159397:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829517.159401:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787829517.159403:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787829517.159406:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d26940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646837632:192.168.203.143@tcp:250:kworker.0 00000100:00000001:1.0:1787829517.159411:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829517.159413:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829517.159414:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787829517.159417:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787829517.159421:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787829517.159424:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787829517.159425:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787829517.159429:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787829517.159431:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787829517.159435:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787829517.159436:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829517.159438:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829517.159445:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787829517.159447:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829517.159450:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787829517.159451:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1787829517.159453:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787829517.159454:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787829517.159456:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829517.159458:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787829517.159460:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829517.159462:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829517.159465:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d27c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646837760:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787829517.159469:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787829517.159471:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787829517.159473:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829517.159474:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787829517.159477:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829517.159483:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787829517.159484:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601351680 : -126234108199936 : ffff8d30d4d27c00) 00000100:00000001:1.0:1787829517.159489:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1787829517.159515:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787829517.159520:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787829517.159523:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829517.159524:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829517.159526:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787829517.159528:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829517.159532:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298990102 00000100:00000001:1.0:1787829517.159535:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298991360 00000100:00000001:1.0:1787829517.159537:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298990102 00000100:00000001:1.0:1787829517.159539:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298991360 00000100:00000001:1.0:1787829517.159541:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829517.159543:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829517.159545:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787829517.159549:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829517/real 1787829517] req@ffff8d30d4d27c00 x1874670646837760/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829542 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787829517.159557:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787829517.159558:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787829517.159561:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787829517.159563:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787829517.159564:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829517.159566:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829517.159567:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787829517.159571:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d27c00 x1874670646837760/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829542 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787829517.159578:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829517.159580:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787829517.159583:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829517.159584:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829517.159585:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829517.159587:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787829517.159589:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787829517.159591:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829517.159595:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787829517.159597:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787829517.159638:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d27c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646837760:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787829517.159643:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787829517.159644:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787829517.159646:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829517.159649:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787829517.159650:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829517.159652:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829517.159656:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d25680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646837888:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787829517.159659:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787829517.159661:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787829517.159663:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829517.159664:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787829517.159668:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829517.159673:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787829517.159674:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601342080 : -126234108209536 : ffff8d30d4d25680) 00000100:00000001:1.0:1787829517.159677:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1787829517.159690:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787829517.159695:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787829517.159697:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829517.159698:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829517.159700:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787829517.159703:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829517.159707:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298990103 00000100:00000001:1.0:1787829517.159710:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298991360 00000100:00000001:1.0:1787829517.159712:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298990103 00000100:00000001:1.0:1787829517.159714:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298991360 00000100:00000001:1.0:1787829517.159716:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829517.159718:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829517.159720:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787829517.159724:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829517/real 1787829517] req@ffff8d30d4d25680 x1874670646837888/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829542 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787829517.159733:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787829517.159734:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787829517.159737:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787829517.159738:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787829517.159740:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829517.159741:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829517.159742:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787829517.159746:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d25680 x1874670646837888/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829542 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787829517.159753:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829517.159755:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787829517.159757:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829517.159759:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829517.159760:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829517.159762:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787829517.159763:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787829517.159766:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829517.159769:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787829517.159771:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787829517.159775:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d25680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646837888:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787829517.159778:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787829517.159780:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787829517.159781:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829517.159783:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787829517.159785:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829517.159787:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829517.159790:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d252c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646838016:192.168.203.143@tcp:8:kworker.0 00000100:00000001:1.0:1787829517.159793:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787829517.159795:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787829517.159796:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829517.159798:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787829517.159801:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829517.159807:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787829517.159808:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601341120 : -126234108210496 : ffff8d30d4d252c0) 00000100:00000001:1.0:1787829517.159811:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1787829517.159823:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787829517.159827:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787829517.159830:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829517.159831:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829517.159832:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787829517.159835:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829517.159838:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298990116 00000100:00000001:1.0:1787829517.159841:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298991360 00000100:00000001:1.0:1787829517.159843:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298990116 00000100:00000001:1.0:1787829517.159844:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298991360 00000100:00000001:1.0:1787829517.159846:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829517.159848:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829517.159850:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787829517.159854:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829517/real 1787829517] req@ffff8d30d4d252c0 x1874670646838016/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829542 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787829517.159861:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787829517.159862:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787829517.159865:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787829517.159866:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787829517.159868:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829517.159870:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829517.159871:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787829517.159874:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d252c0 x1874670646838016/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829542 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787829517.159881:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829517.159883:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787829517.159885:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829517.159886:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829517.159887:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829517.159889:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787829517.159891:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787829517.159893:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829517.159896:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787829517.159899:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787829517.159901:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d252c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646838016:192.168.203.143@tcp:8:kworker.0 00000100:00000001:1.0:1787829517.159905:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829517.159907:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829517.159908:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787829517.159910:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787829517.159912:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787829517.159914:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787829517.159915:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787829517.159917:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787829517.159918:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787829517.159921:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787829517.159922:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829517.159924:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829517.159925:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787829517.159926:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787829517.159928:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787829517.159929:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787829517.159930:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787829517.159931:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787829517.159932:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787829517.159936:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787829517.159937:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829517.159939:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829517.159940:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787829517.159941:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787829517.159943:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787829517.159944:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787829517.159946:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787829517.159947:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787829517.159948:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787829517.159950:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787829517.159951:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829517.159952:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829517.159955:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787829517.159957:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829517.159959:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787829517.159960:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829518.054739:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829518.054747:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829518.054822:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829518.054827:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829518.054832:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829519.078693:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829519.078697:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829519.078747:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829519.078750:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829519.078752:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829520.102669:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829520.102674:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829520.102711:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829520.102714:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829520.102717:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829521.126686:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829521.126690:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829521.126740:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829521.126745:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829521.126747:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829522.150706:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829522.150713:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829522.150759:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829522.150762:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829522.150764:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787829522.278715:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829522.278725:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1787829522.278728:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787829522.278731:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787829522.278735:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787829522.278742:0:468:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 4322 00000100:00080000:3.0:1787829522.278752:0:468:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787829522.278755:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829522.278757:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:3.0:1787829522.278760:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1787829522.278762:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1787829522.278764:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:3.0:1787829522.278766:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:3.0:1787829522.278767:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829522.278768:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:3.0:1787829522.278771:0:468:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:3.0:1787829522.278773:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829522.278775:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787829522.278777:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829522.278779:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829522.278780:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:3.0:1787829522.278782:0:468:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829522.278792:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787829522.278793:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829522.278795:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787829522.278798:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787829522.278800:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829522.278808:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829522.278815:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c7014b40 x1874670646838144/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787829522.278825:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787829522.278829:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829522.278842:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787829522.278845:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829522.278848:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1787829522.278850:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787829522.278851:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787829522.278854:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787829522.278856:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4322 00000100:00080000:3.0:1787829522.278860:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787829522.278862:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829522.278863:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:3.0:1787829522.278866:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1787829522.278867:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1787829522.278869:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000020:00000001:3.0:1787829522.278872:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:1.0:1787829522.278874:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1787829522.278874:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829522.278876:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:1.0:1787829522.278880:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00080000:3.0:1787829522.278882:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:1.0:1787829522.278884:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787829522.278887:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829522.278888:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 02000000:00000001:3.0:1787829522.278891:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1787829522.278893:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829522.278897:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787829522.278899:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829522.278901:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829522.278904:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829522.278906:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829522.278908:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00100000:1.0:1787829522.278913:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c7014b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646838144:192.168.203.143@tcp:250:kworker.0 00000008:00100000:3.0:1787829522.278913:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1787829522.278919:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829522.278921:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:3.0:1787829522.278923:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829522.278926:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1787829522.278929:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1787829522.278930:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829522.278932:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829522.278934:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:3.0:1787829522.278937:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1787829522.278940:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829522.278940:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787829522.278943:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829522.278950:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829522.278956:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00100000:3.0:1787829522.278956:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c70152c0 x1874670646838272/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787829522.278960:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369538368 : -126234340013248 : ffff8d30c7014b40) 00000100:00000001:1.0:1787829522.278966:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:3.0:1787829522.278967:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787829522.278970:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829522.278972:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787829522.278974:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829522.278977:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1787829522.278978:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787829522.278979:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787829522.278980:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787829522.278982:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4322 00000100:00080000:3.0:1787829522.278984:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787829522.278985:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829522.278986:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:3.0:1787829522.278987:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1787829522.278988:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1787829522.278990:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000020:00000001:3.0:1787829522.278991:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:3.0:1787829522.278992:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829522.278993:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:3.0:1787829522.278996:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:3.0:1787829522.278998:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829522.278999:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787829522.279001:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829522.279002:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829522.279004:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:3.0:1787829522.279006:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000800:00000100:1.0:1787829522.279008:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000008:00000001:3.0:1787829522.279009:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829522.279012:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829522.279016:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787829522.279018:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829522.279020:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:1.0:1787829522.279021:0:2390:0:(events.c:34:request_out_callback()) Process entered 02000000:00000001:3.0:1787829522.279022:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787829522.279025:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829522.279026:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829522.279030:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829522.279031:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829522.279032:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787829522.279039:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829522.279039:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c7015680 x1874670646838400/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787829522.279044:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4298996480 00000100:00000001:3.0:1787829522.279053:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787829522.279055:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 00000100:00000001:3.0:1787829522.279056:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829522.279061:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787829522.279063:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1787829522.279065:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4298996480 00000100:00080000:3.0:1787829522.279066:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1787829522.279068:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787829522.279070:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787829522.279073:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 00000100:00000001:3.0:1787829522.279073:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787829522.279076:0:468:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4322 00000100:00080000:3.0:1787829522.279080:0:468:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1787829522.279081:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829522.279082:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787829522.279085:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829522.279085:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:3.0:1787829522.279090:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:1.0:1787829522.279091:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000020:00000001:3.0:1787829522.279092:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00100000:1.0:1787829522.279098:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829522/real 1787829522] req@ffff8d30c7014b40 x1874670646838144/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829547 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:3.0:1787829522.279105:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000020:00000001:3.0:1787829522.279110:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000400:00000001:1.0:1787829522.279111:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787829522.279114:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829522.279115:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829522.279118:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:1.0:1787829522.279119:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787829522.279123:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00080000:3.0:1787829522.279126:0:468:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:1.0:1787829522.279127:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829522.279131:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829522.279132:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829522.279134:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:3.0:1787829522.279136:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00100000:1.0:1787829522.279141:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c7014b40 x1874670646838144/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829547 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1787829522.279145:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829522.279149:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829522.279152:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:1.0:1787829522.279153:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000008:00100000:3.0:1787829522.279154:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00000001:1.0:1787829522.279158:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000008:00000001:3.0:1787829522.279158:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829522.279159:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829522.279165:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:3.0:1787829522.279165:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787829522.279167:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829522.279169:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829522.279170:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:1.0:1787829522.279172:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:3.0:1787829522.279173:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787829522.279175:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829522.279177:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829522.279180:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829522.279181:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787829522.279187:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:3.0:1787829522.279187:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c70143c0 x1874670646838528/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829522.279195:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829522.279197:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787829522.279199:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000001:3.0:1787829522.279200:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829522.279203:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829522.279207:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c7014b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646838144:192.168.203.143@tcp:250:kworker.0 00000100:00000001:1.0:1787829522.279212:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829522.279215:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829522.279216:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787829522.279218:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787829522.279222:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787829522.279224:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787829522.279226:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787829522.279229:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787829522.279231:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787829522.279234:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787829522.279235:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829522.279237:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829522.279242:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787829522.279244:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829522.279246:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787829522.279247:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1787829522.279248:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787829522.279250:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787829522.279251:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829522.279253:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787829522.279254:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829522.279256:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829522.279259:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c70152c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646838272:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787829522.279262:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787829522.279264:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787829522.279265:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829522.279267:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787829522.279269:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829522.279274:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787829522.279275:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369540288 : -126234340011328 : ffff8d30c70152c0) 00000100:00000001:1.0:1787829522.279279:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1787829522.279290:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787829522.279293:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787829522.279295:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829522.279297:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829522.279298:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787829522.279300:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829522.279304:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298995222 00000100:00000001:1.0:1787829522.279306:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298996480 00000100:00000001:1.0:1787829522.279308:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298995222 00000100:00000001:1.0:1787829522.279310:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298996480 00000100:00000001:1.0:1787829522.279312:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829522.279314:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829522.279316:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787829522.279320:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829522/real 1787829522] req@ffff8d30c70152c0 x1874670646838272/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829547 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787829522.279327:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787829522.279328:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787829522.279330:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787829522.279331:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787829522.279332:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829522.279334:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829522.279335:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787829522.279338:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c70152c0 x1874670646838272/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829547 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787829522.279345:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829522.279346:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787829522.279349:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829522.279351:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829522.279352:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829522.279353:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787829522.279355:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787829522.279357:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829522.279359:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787829522.279361:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787829522.279364:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c70152c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646838272:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787829522.279367:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787829522.279368:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787829522.279369:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829522.279371:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787829522.279372:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829522.279374:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829522.279377:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c7015680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646838400:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787829522.279379:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787829522.279381:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787829522.279382:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829522.279384:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787829522.279386:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829522.279390:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787829522.279391:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369541248 : -126234340010368 : ffff8d30c7015680) 00000100:00000001:1.0:1787829522.279394:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1787829522.279402:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787829522.279406:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787829522.279407:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829522.279409:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829522.279410:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787829522.279412:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829522.279415:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298995222 00000100:00000001:1.0:1787829522.279417:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298996480 00000100:00000001:1.0:1787829522.279419:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298995222 00000100:00000001:1.0:1787829522.279420:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298996480 00000100:00000001:1.0:1787829522.279422:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829522.279425:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829522.279427:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787829522.279430:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829522/real 1787829522] req@ffff8d30c7015680 x1874670646838400/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829547 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787829522.279436:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787829522.279437:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787829522.279439:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787829522.279440:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787829522.279441:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829522.279443:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829522.279444:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787829522.279447:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c7015680 x1874670646838400/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829547 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787829522.279453:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829522.279455:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787829522.279457:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829522.279459:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829522.279460:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829522.279461:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787829522.279462:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787829522.279464:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829522.279483:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787829522.279485:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787829522.279488:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c7015680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646838400:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787829522.279492:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787829522.279493:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787829522.279495:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829522.279497:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787829522.279499:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829522.279501:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829522.279504:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c70143c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646838528:192.168.203.143@tcp:8:kworker.0 00000100:00000001:1.0:1787829522.279507:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787829522.279514:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787829522.279516:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829522.279518:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787829522.279520:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829522.279526:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787829522.279527:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369536448 : -126234340015168 : ffff8d30c70143c0) 00000100:00000001:1.0:1787829522.279530:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1787829522.279539:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787829522.279543:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787829522.279545:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829522.279546:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829522.279547:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787829522.279549:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829522.279552:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4298995235 00000100:00000001:1.0:1787829522.279555:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4298996480 00000100:00000001:1.0:1787829522.279557:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4298995235 00000100:00000001:1.0:1787829522.279559:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4298996480 00000100:00000001:1.0:1787829522.279561:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829522.279562:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829522.279564:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787829522.279567:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829522/real 1787829522] req@ffff8d30c70143c0 x1874670646838528/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829547 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787829522.279574:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787829522.279575:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787829522.279577:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787829522.279578:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787829522.279580:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829522.279582:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829522.279583:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787829522.279586:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c70143c0 x1874670646838528/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829547 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787829522.279597:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829522.279621:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787829522.279625:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829522.279626:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829522.279627:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829522.279629:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787829522.279630:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787829522.279633:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829522.279636:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787829522.279638:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787829522.279641:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c70143c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646838528:192.168.203.143@tcp:8:kworker.0 00000100:00000001:1.0:1787829522.279645:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829522.279646:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829522.279653:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787829522.279655:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787829522.279657:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787829522.279658:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787829522.279660:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787829522.279662:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787829522.279663:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787829522.279664:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787829522.279666:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829522.279667:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829522.279669:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787829522.279670:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787829522.279671:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787829522.279672:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787829522.279673:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787829522.279675:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787829522.279676:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787829522.279677:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787829522.279678:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829522.279679:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829522.279680:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787829522.279681:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787829522.279683:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787829522.279684:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787829522.279685:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787829522.279686:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787829522.279687:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787829522.279688:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787829522.279689:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829522.279691:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829522.279693:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787829522.279695:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829522.279696:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787829522.279697:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829523.174711:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829523.174718:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829523.174766:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829523.174769:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829523.174771:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829524.198652:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829524.198658:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829524.198703:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829524.198707:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829524.198710:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829525.222666:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829525.222671:0:815:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1787829525.222673:0:815:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1787829525.222675:0:815:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829525.222678:0:815:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1787829525.222680:0:815:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829525.222682:0:815:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829525.222683:0:815:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829525.222724:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829525.222726:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829525.222729:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829526.246661:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829526.246667:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829526.246724:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829526.246728:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829526.246731:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829527.270671:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829527.270676:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829527.270706:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829527.270709:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829527.270712:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787829527.398672:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829527.398680:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1787829527.398682:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787829527.398686:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787829527.398690:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787829527.398696:0:468:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 4327 00000100:00080000:3.0:1787829527.398705:0:468:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787829527.398708:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829527.398710:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:3.0:1787829527.398714:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1787829527.398716:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1787829527.398719:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:3.0:1787829527.398722:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:3.0:1787829527.398724:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829527.398725:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:3.0:1787829527.398729:0:468:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:3.0:1787829527.398731:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829527.398734:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787829527.398736:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829527.398739:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829527.398741:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:3.0:1787829527.398743:0:468:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829527.398752:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787829527.398754:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829527.398757:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787829527.398759:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787829527.398763:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829527.398773:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829527.398780:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c7014f00 x1874670646838656/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787829527.398789:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787829527.398793:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829527.398807:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787829527.398809:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829527.398812:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1787829527.398813:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787829527.398815:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787829527.398817:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787829527.398819:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4327 00000100:00080000:3.0:1787829527.398823:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787829527.398824:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829527.398826:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:3.0:1787829527.398828:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1787829527.398829:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1787829527.398831:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000020:00000001:3.0:1787829527.398833:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:3.0:1787829527.398835:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829527.398836:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:3.0:1787829527.398840:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:3.0:1787829527.398842:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829527.398844:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787829527.398849:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829527.398851:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:3.0:1787829527.398851:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829527.398856:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:1.0:1787829527.398857:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000008:00100000:3.0:1787829527.398860:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00000001:1.0:1787829527.398861:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787829527.398864:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000008:00000001:3.0:1787829527.398865:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829527.398868:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829527.398869:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829527.398874:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787829527.398875:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787829527.398877:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829527.398878:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829527.398880:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1787829527.398883:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829527.398883:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787829527.398886:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829527.398891:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c7014f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646838656:192.168.203.143@tcp:250:kworker.0 00000100:00000001:3.0:1787829527.398892:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829527.398899:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00100000:3.0:1787829527.398902:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c70161c0 x1874670646838784/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:1.0:1787829527.398904:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787829527.398907:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829527.398911:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:3.0:1787829527.398913:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787829527.398916:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829527.398917:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829527.398919:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787829527.398923:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829527.398926:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1787829527.398928:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787829527.398929:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787829527.398931:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787829527.398932:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:1.0:1787829527.398934:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369539328 : -126234340012288 : ffff8d30c7014f00) 00000100:00080000:3.0:1787829527.398937:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4327 00000100:00000001:1.0:1787829527.398941:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00080000:3.0:1787829527.398942:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787829527.398945:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829527.398946:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:3.0:1787829527.398948:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1787829527.398950:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1787829527.398951:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000020:00000001:3.0:1787829527.398953:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:3.0:1787829527.398956:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829527.398957:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:3.0:1787829527.398959:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:3.0:1787829527.398961:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829527.398963:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787829527.398965:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829527.398966:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829527.398968:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:3.0:1787829527.398970:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000800:00000100:1.0:1787829527.398972:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000008:00000001:3.0:1787829527.398974:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829527.398976:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829527.398980:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787829527.398981:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829527.398982:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787829527.398984:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787829527.398986:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:1.0:1787829527.398987:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 02000000:00000001:3.0:1787829527.398988:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829527.398990:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829527.398992:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1787829527.398994:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829527.398997:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829527.399001:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4299001600 00000100:00100000:3.0:1787829527.399001:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c7015a40 x1874670646838912/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787829527.399009:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 00000100:00000001:3.0:1787829527.399010:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787829527.399013:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829527.399016:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787829527.399018:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829527.399021:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1787829527.399022:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4299001600 00000100:00000001:3.0:1787829527.399023:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787829527.399026:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787829527.399028:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 00000100:00000001:3.0:1787829527.399029:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:1.0:1787829527.399031:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829527.399035:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787829527.399035:0:468:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4327 00000100:00000001:1.0:1787829527.399040:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00080000:3.0:1787829527.399042:0:468:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787829527.399046:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00100000:1.0:1787829527.399047:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829527/real 1787829527] req@ffff8d30c7014f00 x1874670646838656/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829552 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829527.399048:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:3.0:1787829527.399054:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1787829527.399056:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1787829527.399059:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000400:00000001:1.0:1787829527.399061:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787829527.399064:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000020:00000001:3.0:1787829527.399064:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:1.0:1787829527.399067:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829527.399069:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787829527.399070:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787829527.399073:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829527.399073:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:1.0:1787829527.399075:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829527.399078:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:3.0:1787829527.399082:0:468:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00100000:1.0:1787829527.399084:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c7014f00 x1874670646838656/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829552 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829527.399086:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829527.399091:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1787829527.399095:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:3.0:1787829527.399095:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829527.399098:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 02000000:00000001:3.0:1787829527.399099:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829527.399101:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829527.399102:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:1.0:1787829527.399103:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829527.399106:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000008:00100000:3.0:1787829527.399106:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00000001:1.0:1787829527.399108:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787829527.399111:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000008:00000001:3.0:1787829527.399111:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829527.399116:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1787829527.399117:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829527.399122:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 02000000:00000001:3.0:1787829527.399123:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:1.0:1787829527.399124:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 02000000:00000001:3.0:1787829527.399126:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829527.399128:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c7014f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646838656:192.168.203.143@tcp:250:kworker.0 00000100:00000001:3.0:1787829527.399130:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:1.0:1787829527.399133:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:3.0:1787829527.399133:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:1.0:1787829527.399137:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 02000000:00000001:3.0:1787829527.399137:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829527.399139:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787829527.399141:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 00000100:00000001:3.0:1787829527.399144:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829527.399148:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787829527.399152:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000100:00100000:3.0:1787829527.399152:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c7016580 x1874670646839040/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000020:00000001:1.0:1787829527.399154:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787829527.399158:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787829527.399160:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787829527.399162:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829527.399164:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787829527.399165:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829527.399168:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829527.399169:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829527.399173:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829527.399174:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787829527.399176:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829527.399177:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787829527.399179:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1787829527.399180:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787829527.399181:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787829527.399182:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829527.399183:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787829527.399184:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829527.399186:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829527.399188:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c70161c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646838784:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787829527.399190:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787829527.399192:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787829527.399193:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829527.399195:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787829527.399197:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829527.399201:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787829527.399203:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369544128 : -126234340007488 : ffff8d30c70161c0) 00000100:00000001:1.0:1787829527.399207:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1787829527.399219:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787829527.399223:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787829527.399225:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829527.399227:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829527.399228:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787829527.399230:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829527.399234:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4299000342 00000100:00000001:1.0:1787829527.399237:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4299001600 00000100:00000001:1.0:1787829527.399239:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4299000342 00000100:00000001:1.0:1787829527.399241:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4299001600 00000100:00000001:1.0:1787829527.399244:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829527.399245:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829527.399247:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787829527.399251:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829527/real 1787829527] req@ffff8d30c70161c0 x1874670646838784/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829552 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787829527.399258:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787829527.399260:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787829527.399262:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787829527.399263:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787829527.399265:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829527.399266:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829527.399268:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787829527.399271:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c70161c0 x1874670646838784/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829552 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787829527.399277:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829527.399279:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787829527.399281:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829527.399283:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829527.399284:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829527.399285:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787829527.399287:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787829527.399289:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829527.399293:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787829527.399295:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787829527.399298:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c70161c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646838784:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787829527.399301:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787829527.399303:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787829527.399304:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829527.399306:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787829527.399308:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829527.399310:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829527.399313:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c7015a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646838912:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787829527.399316:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787829527.399317:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787829527.399319:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829527.399320:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787829527.399323:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829527.399327:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787829527.399328:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369542208 : -126234340009408 : ffff8d30c7015a40) 00000100:00000001:1.0:1787829527.399331:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1787829527.399341:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787829527.399345:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787829527.399347:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829527.399349:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829527.399350:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787829527.399353:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829527.399355:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4299000342 00000100:00000001:1.0:1787829527.399358:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4299001600 00000100:00000001:1.0:1787829527.399360:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4299000342 00000100:00000001:1.0:1787829527.399362:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4299001600 00000100:00000001:1.0:1787829527.399363:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829527.399366:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829527.399367:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787829527.399371:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829527/real 1787829527] req@ffff8d30c7015a40 x1874670646838912/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829552 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787829527.399379:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787829527.399380:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787829527.399382:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787829527.399383:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787829527.399384:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829527.399385:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829527.399386:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787829527.399388:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c7015a40 x1874670646838912/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829552 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787829527.399392:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829527.399394:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787829527.399395:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829527.399396:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829527.399396:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829527.399397:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787829527.399398:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787829527.399399:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829527.399401:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787829527.399402:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787829527.399404:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c7015a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646838912:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787829527.399406:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787829527.399407:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787829527.399407:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829527.399408:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787829527.399409:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829527.399411:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829527.399412:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c7016580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646839040:192.168.203.143@tcp:8:kworker.0 00000100:00000001:1.0:1787829527.399414:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787829527.399415:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787829527.399416:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829527.399417:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787829527.399419:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829527.399423:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787829527.399424:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369545088 : -126234340006528 : ffff8d30c7016580) 00000100:00000001:1.0:1787829527.399427:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1787829527.399453:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787829527.399457:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787829527.399459:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829527.399460:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829527.399462:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787829527.399465:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829527.399468:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4299000355 00000100:00000001:1.0:1787829527.399470:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4299001600 00000100:00000001:1.0:1787829527.399472:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4299000355 00000100:00000001:1.0:1787829527.399474:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4299001600 00000100:00000001:1.0:1787829527.399476:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829527.399478:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829527.399480:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787829527.399484:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829527/real 1787829527] req@ffff8d30c7016580 x1874670646839040/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829552 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787829527.399491:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787829527.399492:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787829527.399495:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787829527.399497:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787829527.399498:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829527.399500:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829527.399501:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787829527.399505:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c7016580 x1874670646839040/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829552 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787829527.399517:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829527.399519:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787829527.399521:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829527.399523:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829527.399524:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829527.399526:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787829527.399527:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787829527.399529:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829527.399532:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787829527.399534:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787829527.399537:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c7016580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646839040:192.168.203.143@tcp:8:kworker.0 00000100:00000001:1.0:1787829527.399541:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829527.399543:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829527.399544:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787829527.399545:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787829527.399547:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787829527.399549:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787829527.399550:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787829527.399552:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787829527.399554:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787829527.399556:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787829527.399557:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829527.399558:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829527.399559:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787829527.399560:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787829527.399563:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787829527.399564:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787829527.399565:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787829527.399566:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787829527.399567:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787829527.399569:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787829527.399570:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829527.399571:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829527.399572:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787829527.399574:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787829527.399575:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787829527.399576:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787829527.399577:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787829527.399579:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787829527.399580:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787829527.399581:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787829527.399582:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829527.399584:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829527.399587:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787829527.399588:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829527.399590:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787829527.399591:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829528.294690:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829528.294696:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829528.294747:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829528.294750:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829528.294753:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829529.318707:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829529.318713:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829529.318765:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829529.318769:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829529.318772:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829530.342732:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829530.342741:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829530.342797:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829530.342802:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829530.342805:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829531.366756:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829531.366767:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829531.366838:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829531.366842:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829531.366845:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829532.390721:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829532.390728:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829532.390803:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829532.390809:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829532.390812:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787829532.518770:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829532.518781:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1787829532.518784:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787829532.518794:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787829532.518799:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787829532.518806:0:468:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 4333 00000100:00080000:2.0:1787829532.518816:0:468:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787829532.518818:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829532.518819:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787829532.518822:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787829532.518824:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787829532.518826:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:2.0:1787829532.518828:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:2.0:1787829532.518829:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829532.518830:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:2.0:1787829532.518833:0:468:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787829532.518834:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829532.518837:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787829532.518849:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787829532.518853:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829532.518855:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:2.0:1787829532.518858:0:468:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829532.518869:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787829532.518871:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829532.518874:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787829532.518876:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787829532.518880:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829532.518889:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787829532.518898:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2f480 x1874670646839168/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787829532.518919:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787829532.518922:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829532.518938:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787829532.518941:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829532.518944:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1787829532.518946:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787829532.518947:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787829532.518950:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787829532.518952:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4333 00000100:00080000:2.0:1787829532.518956:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787829532.518958:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829532.518960:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787829532.518963:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787829532.518965:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787829532.518967:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000020:00000001:2.0:1787829532.518969:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:2.0:1787829532.518971:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829532.518973:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:1.0:1787829532.518975:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00080000:2.0:1787829532.518978:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:1.0:1787829532.518979:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1787829532.518982:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787829532.518985:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787829532.518985:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829532.518987:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829532.518989:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1787829532.518992:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787829532.518995:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787829532.518996:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829532.518997:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829532.519003:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646839168:192.168.203.143@tcp:250:kworker.0 02000000:00000001:2.0:1787829532.519003:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829532.519007:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:1.0:1787829532.519009:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000008:00100000:2.0:1787829532.519012:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 02000000:00000001:1.0:1787829532.519014:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000008:00000001:2.0:1787829532.519015:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829532.519018:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829532.519018:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829532.519023:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787829532.519027:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829532.519036:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787829532.519039:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829532.519042:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787829532.519044:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:1.0:1787829532.519046:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 02000000:00000001:2.0:1787829532.519046:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829532.519050:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417881728 : -126234291669888 : ffff8d30c9e2f480) 00000100:00000001:1.0:1787829532.519055:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:2.0:1787829532.519055:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787829532.519064:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2da40 x1874670646839296/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787829532.519072:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787829532.519074:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829532.519076:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787829532.519078:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829532.519080:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1787829532.519082:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787829532.519083:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787829532.519084:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787829532.519086:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4333 00000100:00080000:2.0:1787829532.519089:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000800:00000100:1.0:1787829532.519092:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:2.0:1787829532.519093:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829532.519096:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787829532.519098:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787829532.519099:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787829532.519101:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000020:00000001:2.0:1787829532.519103:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:1.0:1787829532.519104:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1787829532.519105:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829532.519108:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:1.0:1787829532.519110:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829532.519113:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829532.519114:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00080000:2.0:1787829532.519117:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:1.0:1787829532.519118:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829532.519124:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4299006720 00000100:00000001:2.0:1787829532.519124:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829532.519128:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1787829532.519131:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 02000000:00000001:2.0:1787829532.519132:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787829532.519137:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829532.519138:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4299006720 00000100:00000001:2.0:1787829532.519139:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:1.0:1787829532.519144:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 00000008:00100000:2.0:1787829532.519144:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00000001:1.0:1787829532.519148:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000008:00000001:2.0:1787829532.519150:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829532.519151:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829532.519156:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829532.519157:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787829532.519162:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829532/real 1787829532] req@ffff8d30c9e2f480 x1874670646839168/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829557 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:2.0:1787829532.519164:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787829532.519168:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829532.519173:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000400:00000001:1.0:1787829532.519175:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787829532.519177:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 02000000:00000001:2.0:1787829532.519177:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:1.0:1787829532.519182:0:2390:0:(events.c:78:reply_in_callback()) Process entered 02000000:00000001:2.0:1787829532.519182:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829532.519185:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787829532.519187:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829532.519188:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829532.519190:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787829532.519192:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2f480 x1874670646839168/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829557 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787829532.519196:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829532.519197:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829532.519202:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787829532.519204:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829532.519206:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829532.519207:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787829532.519207:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2c000 x1874670646839424/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787829532.519211:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787829532.519213:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787829532.519218:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829532.519224:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:2.0:1787829532.519225:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787829532.519226:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000001:2.0:1787829532.519229:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829532.519232:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646839168:192.168.203.143@tcp:250:kworker.0 00000100:00000001:2.0:1787829532.519234:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829532.519237:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:2.0:1787829532.519238:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1787829532.519241:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00080000:2.0:1787829532.519243:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1787829532.519244:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 00000100:00000001:2.0:1787829532.519245:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 02000000:00000001:1.0:1787829532.519247:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 00000100:00080000:2.0:1787829532.519248:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 02000000:00000001:1.0:1787829532.519251:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00000001:2.0:1787829532.519252:0:468:0:(import.c:485:import_select_connection()) Process entered 00000020:00000001:1.0:1787829532.519255:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000100:00080000:2.0:1787829532.519257:0:468:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4333 00000020:00000001:1.0:1787829532.519258:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787829532.519262:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787829532.519264:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00080000:2.0:1787829532.519264:0:468:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787829532.519268:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787829532.519269:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787829532.519271:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829532.519271:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:1.0:1787829532.519274:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:2.0:1787829532.519277:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787829532.519280:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:1.0:1787829532.519281:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000020:00000001:2.0:1787829532.519282:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:1.0:1787829532.519283:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829532.519287:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:2.0:1787829532.519288:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:1.0:1787829532.519289:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1787829532.519291:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787829532.519294:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787829532.519294:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787829532.519295:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829532.519299:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:2.0:1787829532.519299:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 02000000:00000001:1.0:1787829532.519301:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829532.519304:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787829532.519307:0:468:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00100000:1.0:1787829532.519308:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2da40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646839296:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787829532.519315:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:2.0:1787829532.519315:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829532.519317:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787829532.519321:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829532.519321:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1787829532.519325:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787829532.519327:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829532.519328:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829532.519330:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 02000000:00000001:2.0:1787829532.519332:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829532.519333:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417875008 : -126234291676608 : ffff8d30c9e2da40) 00000100:00000001:2.0:1787829532.519337:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:1.0:1787829532.519339:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000008:00100000:2.0:1787829532.519340:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000008:00000001:2.0:1787829532.519343:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829532.519345:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829532.519349:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000800:00000100:1.0:1787829532.519352:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 02000000:00000001:2.0:1787829532.519352:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829532.519354:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787829532.519356:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:1.0:1787829532.519357:0:2390:0:(events.c:34:request_out_callback()) Process entered 02000000:00000001:2.0:1787829532.519358:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829532.519361:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829532.519363:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829532.519364:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829532.519365:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787829532.519367:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829532.519371:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4299005462 00000100:00100000:2.0:1787829532.519374:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2e940 x1874670646839552/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787829532.519375:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4299006720 00000100:00000001:1.0:1787829532.519378:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4299005462 00000100:00000001:1.0:1787829532.519382:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4299006720 00000100:00000001:1.0:1787829532.519385:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829532.519389:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829532.519391:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787829532.519407:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829532/real 1787829532] req@ffff8d30c9e2da40 x1874670646839296/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829557 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787829532.519408:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787829532.519413:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:1.0:1787829532.519419:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000001:2.0:1787829532.519419:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:1.0:1787829532.519421:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787829532.519425:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787829532.519427:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787829532.519428:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829532.519430:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829532.519431:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787829532.519434:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2da40 x1874670646839296/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829557 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787829532.519439:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829532.519440:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787829532.519442:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829532.519443:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829532.519444:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829532.519445:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787829532.519447:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787829532.519448:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829532.519450:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787829532.519452:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787829532.519454:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2da40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646839296:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787829532.519456:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787829532.519457:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787829532.519458:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829532.519459:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787829532.519460:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829532.519461:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829532.519462:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646839424:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787829532.519464:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787829532.519465:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787829532.519466:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829532.519467:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787829532.519469:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829532.519472:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787829532.519473:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417868288 : -126234291683328 : ffff8d30c9e2c000) 00000100:00000001:1.0:1787829532.519475:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1787829532.519482:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787829532.519485:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787829532.519486:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829532.519487:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829532.519488:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787829532.519489:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829532.519492:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4299005462 00000100:00000001:1.0:1787829532.519493:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4299006720 00000100:00000001:1.0:1787829532.519495:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4299005462 00000100:00000001:1.0:1787829532.519496:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4299006720 00000100:00000001:1.0:1787829532.519497:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829532.519498:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829532.519499:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787829532.519501:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829532/real 1787829532] req@ffff8d30c9e2c000 x1874670646839424/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829557 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787829532.519510:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787829532.519511:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787829532.519512:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787829532.519513:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787829532.519514:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829532.519515:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829532.519515:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787829532.519517:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2c000 x1874670646839424/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829557 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787829532.519521:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829532.519522:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787829532.519524:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829532.519524:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829532.519525:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829532.519526:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787829532.519527:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787829532.519528:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829532.519530:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787829532.519531:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787829532.519532:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646839424:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787829532.519534:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829532.519535:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829532.519536:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787829532.519536:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787829532.519538:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787829532.519538:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787829532.519539:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787829532.519540:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787829532.519541:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787829532.519543:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787829532.519544:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829532.519545:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829532.519546:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787829532.519547:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787829532.519547:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787829532.519548:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787829532.519549:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787829532.519550:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787829532.519550:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787829532.519552:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787829532.519552:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829532.519553:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829532.519555:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787829532.519556:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829532.519557:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787829532.519557:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1787829532.519558:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787829532.519559:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787829532.519560:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829532.519561:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787829532.519562:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829532.519563:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829532.519564:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646839552:192.168.203.143@tcp:8:kworker.0 00000100:00000001:1.0:1787829532.519566:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787829532.519567:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787829532.519568:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829532.519569:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787829532.519570:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829532.519573:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787829532.519574:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417878848 : -126234291672768 : ffff8d30c9e2e940) 00000100:00000001:1.0:1787829532.519576:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1787829532.519581:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787829532.519583:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787829532.519584:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829532.519585:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829532.519586:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787829532.519587:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829532.519589:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4299005475 00000100:00000001:1.0:1787829532.519591:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4299006720 00000100:00000001:1.0:1787829532.519593:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4299005475 00000100:00000001:1.0:1787829532.519594:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4299006720 00000100:00000001:1.0:1787829532.519595:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829532.519596:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829532.519597:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787829532.519627:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829532/real 1787829532] req@ffff8d30c9e2e940 x1874670646839552/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829557 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787829532.519632:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787829532.519633:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787829532.519634:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787829532.519635:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787829532.519636:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829532.519637:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829532.519638:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787829532.519641:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2e940 x1874670646839552/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829557 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787829532.519645:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829532.519646:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787829532.519647:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829532.519648:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829532.519649:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829532.519649:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787829532.519650:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787829532.519652:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829532.519653:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787829532.519654:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787829532.519656:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646839552:192.168.203.143@tcp:8:kworker.0 00000100:00000001:1.0:1787829532.519658:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829532.519659:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829532.519660:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787829532.519661:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787829532.519662:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787829532.519663:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787829532.519663:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787829532.519665:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787829532.519665:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787829532.519668:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787829532.519669:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829532.519670:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829532.519672:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787829532.519672:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829532.519673:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787829532.519674:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829533.414696:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829533.414700:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829533.414782:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829533.414788:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829533.414791:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829534.438685:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829534.438692:0:815:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1787829534.438693:0:815:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1787829534.438696:0:815:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829534.438700:0:815:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1787829534.438702:0:815:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829534.438704:0:815:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829534.438706:0:815:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829534.438755:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829534.438758:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829534.438760:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829535.462661:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829535.462666:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829535.462716:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829535.462720:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829535.462723:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829536.486682:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829536.486689:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829536.486739:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829536.486742:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829536.486744:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829537.510666:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829537.510670:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829537.510714:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829537.510718:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829537.510722:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787829537.638683:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829537.638689:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1787829537.638691:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787829537.638694:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787829537.638697:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787829537.638702:0:468:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 4338 00000100:00080000:3.0:1787829537.638709:0:468:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787829537.638711:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829537.638713:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:3.0:1787829537.638715:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1787829537.638717:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1787829537.638719:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:3.0:1787829537.638721:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:3.0:1787829537.638722:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829537.638723:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:3.0:1787829537.638726:0:468:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:3.0:1787829537.638728:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829537.638730:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787829537.638732:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829537.638734:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829537.638735:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:3.0:1787829537.638736:0:468:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829537.638746:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787829537.638747:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829537.638749:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787829537.638750:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787829537.638753:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829537.638768:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829537.638775:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c7016580 x1874670646839680/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787829537.638781:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787829537.638784:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829537.638793:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787829537.638795:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829537.638797:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1787829537.638798:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787829537.638799:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787829537.638801:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787829537.638802:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4338 00000100:00080000:3.0:1787829537.638804:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787829537.638805:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829537.638806:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:3.0:1787829537.638808:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1787829537.638810:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1787829537.638812:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000020:00000001:3.0:1787829537.638814:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:3.0:1787829537.638816:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829537.638818:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:1.0:1787829537.638821:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00080000:3.0:1787829537.638822:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:1.0:1787829537.638825:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787829537.638826:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829537.638828:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 02000000:00000001:3.0:1787829537.638830:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1787829537.638831:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787829537.638833:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829537.638836:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829537.638837:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787829537.638840:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829537.638841:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829537.638842:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829537.638846:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00100000:1.0:1787829537.638848:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c7016580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646839680:192.168.203.143@tcp:250:kworker.0 00000008:00100000:3.0:1787829537.638851:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00000001:1.0:1787829537.638853:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000008:00000001:3.0:1787829537.638855:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829537.638857:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:3.0:1787829537.638858:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829537.638860:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829537.638863:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1787829537.638865:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1787829537.638867:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829537.638875:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829537.638877:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:1.0:1787829537.638878:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 02000000:00000001:3.0:1787829537.638879:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:1.0:1787829537.638881:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369545088 : -126234340006528 : ffff8d30c7016580) 02000000:00000001:3.0:1787829537.638883:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829537.638884:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:3.0:1787829537.638892:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829537.638900:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c7015a40 x1874670646839808/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787829537.638907:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787829537.638909:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:1.0:1787829537.638913:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:3.0:1787829537.638913:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787829537.638919:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829537.638923:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:1.0:1787829537.638925:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787829537.638925:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:1.0:1787829537.638930:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829537.638932:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829537.638934:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00080000:3.0:1787829537.638936:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787829537.638937:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829537.638940:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:1.0:1787829537.638942:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4299011840 00000100:00080000:3.0:1787829537.638946:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4338 00000100:00080000:3.0:1787829537.638950:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1787829537.638951:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 00000100:00000001:3.0:1787829537.638952:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829537.638954:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:3.0:1787829537.638957:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1787829537.638958:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1787829537.638960:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000020:00000001:3.0:1787829537.638961:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:3.0:1787829537.638962:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829537.638963:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:1.0:1787829537.638964:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4299011840 00000100:00080000:3.0:1787829537.638966:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:1.0:1787829537.638969:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 00000100:00000001:3.0:1787829537.638969:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829537.638973:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1787829537.638974:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829537.638976:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829537.638977:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829537.638980:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829537.638981:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00000001:3.0:1787829537.638983:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:3.0:1787829537.638986:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829537.638988:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829537/real 1787829537] req@ffff8d30c7016580 x1874670646839680/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829562 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00000001:3.0:1787829537.638989:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829537.638992:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829537.638997:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787829537.638999:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:1.0:1787829537.639001:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000001:3.0:1787829537.639001:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000400:00000001:1.0:1787829537.639005:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 02000000:00000001:3.0:1787829537.639005:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787829537.639008:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829537.639009:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787829537.639012:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787829537.639013:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829537.639013:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829537.639018:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829537.639020:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829537.639022:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c70161c0 x1874670646839936/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787829537.639024:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c7016580 x1874670646839680/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829562 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787829537.639046:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829537.639048:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787829537.639049:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829537.639051:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829537.639053:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829537.639055:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829537.639056:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829537.639058:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:3.0:1787829537.639058:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1787829537.639060:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787829537.639062:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829537.639063:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1787829537.639067:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00000001:3.0:1787829537.639067:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787829537.639069:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1787829537.639072:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829537.639073:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:1.0:1787829537.639074:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00080000:3.0:1787829537.639077:0:468:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4338 00000100:00100000:1.0:1787829537.639080:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c7016580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646839680:192.168.203.143@tcp:250:kworker.0 00000100:00080000:3.0:1787829537.639083:0:468:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1787829537.639084:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829537.639089:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829537.639089:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787829537.639091:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 00000100:00000001:3.0:1787829537.639092:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 02000000:00000001:1.0:1787829537.639093:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787829537.639096:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829537.639097:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:1.0:1787829537.639099:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829537.639100:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:1.0:1787829537.639101:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 00000020:00000001:3.0:1787829537.639104:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 02000000:00000001:1.0:1787829537.639106:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787829537.639108:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787829537.639110:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000020:00000001:3.0:1787829537.639110:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:1.0:1787829537.639112:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829537.639115:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829537.639116:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829537.639119:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:1.0:1787829537.639121:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787829537.639123:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829537.639125:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00080000:3.0:1787829537.639125:0:468:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:1.0:1787829537.639128:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1787829537.639131:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787829537.639132:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787829537.639132:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829537.639134:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829537.639136:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1787829537.639137:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787829537.639140:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829537.639141:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829537.639141:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00100000:1.0:1787829537.639144:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c7015a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646839808:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787829537.639147:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787829537.639148:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787829537.639149:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829537.639151:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829537.639152:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:3.0:1787829537.639155:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 02000000:00000001:1.0:1787829537.639156:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000008:00100000:3.0:1787829537.639158:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00000001:1.0:1787829537.639161:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787829537.639163:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369542208 : -126234340009408 : ffff8d30c7015a40) 00000008:00000001:3.0:1787829537.639163:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829537.639168:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829537.639169:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 02000000:00000001:3.0:1787829537.639176:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787829537.639179:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:1.0:1787829537.639181:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:3.0:1787829537.639181:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787829537.639183:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:1.0:1787829537.639186:0:2390:0:(events.c:34:request_out_callback()) Process entered 02000000:00000001:3.0:1787829537.639187:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829537.639189:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829537.639191:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829537.639192:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787829537.639194:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829537.639196:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829537.639197:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4299010582 00000100:00000001:1.0:1787829537.639201:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4299011840 00000100:00000001:1.0:1787829537.639203:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4299010582 00000100:00000001:1.0:1787829537.639205:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4299011840 00000100:00100000:3.0:1787829537.639206:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c7014f00 x1874670646840064/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787829537.639207:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829537.639210:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829537.639213:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787829537.639218:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829537/real 1787829537] req@ffff8d30c7015a40 x1874670646839808/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829562 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829537.639220:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787829537.639223:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829537.639227:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:1.0:1787829537.639228:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787829537.639230:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787829537.639232:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787829537.639235:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787829537.639236:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829537.639237:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829537.639238:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787829537.639240:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c7015a40 x1874670646839808/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829562 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787829537.639245:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829537.639246:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787829537.639247:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829537.639248:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829537.639249:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829537.639250:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787829537.639251:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787829537.639252:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829537.639255:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787829537.639256:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787829537.639258:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c7015a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646839808:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787829537.639260:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787829537.639261:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787829537.639262:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829537.639263:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787829537.639264:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829537.639265:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829537.639266:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c70161c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646839936:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787829537.639268:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787829537.639269:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787829537.639270:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829537.639271:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787829537.639273:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829537.639275:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787829537.639276:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369544128 : -126234340007488 : ffff8d30c70161c0) 00000100:00000001:1.0:1787829537.639278:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1787829537.639283:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787829537.639286:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787829537.639287:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829537.639288:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829537.639289:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787829537.639290:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829537.639292:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4299010582 00000100:00000001:1.0:1787829537.639294:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4299011840 00000100:00000001:1.0:1787829537.639295:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4299010582 00000100:00000001:1.0:1787829537.639296:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4299011840 00000100:00000001:1.0:1787829537.639298:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829537.639299:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829537.639299:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787829537.639302:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829537/real 1787829537] req@ffff8d30c70161c0 x1874670646839936/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829562 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787829537.639306:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787829537.639307:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787829537.639308:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787829537.639309:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787829537.639310:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829537.639310:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829537.639311:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787829537.639313:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c70161c0 x1874670646839936/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829562 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787829537.639317:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829537.639318:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787829537.639320:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829537.639320:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829537.639321:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829537.639322:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787829537.639323:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787829537.639324:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829537.639326:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787829537.639327:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787829537.639328:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c70161c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646839936:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787829537.639330:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829537.639331:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829537.639332:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787829537.639332:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787829537.639334:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787829537.639334:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787829537.639335:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787829537.639336:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787829537.639337:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787829537.639338:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787829537.639339:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829537.639340:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829537.639340:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787829537.639341:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787829537.639342:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787829537.639343:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787829537.639343:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787829537.639344:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787829537.639345:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787829537.639345:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787829537.639346:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829537.639347:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829537.639348:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787829537.639349:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829537.639350:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787829537.639351:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1787829537.639352:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787829537.639352:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787829537.639353:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829537.639354:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787829537.639355:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829537.639356:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829537.639369:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c7014f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646840064:192.168.203.143@tcp:8:kworker.0 00000100:00000001:1.0:1787829537.639372:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787829537.639373:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787829537.639374:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829537.639375:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787829537.639377:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829537.639379:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787829537.639380:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369539328 : -126234340012288 : ffff8d30c7014f00) 00000100:00000001:1.0:1787829537.639382:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1787829537.639389:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787829537.639391:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787829537.639393:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829537.639393:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829537.639394:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787829537.639396:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829537.639398:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4299010595 00000100:00000001:1.0:1787829537.639399:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4299011840 00000100:00000001:1.0:1787829537.639400:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4299010595 00000100:00000001:1.0:1787829537.639402:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4299011840 00000100:00000001:1.0:1787829537.639403:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829537.639404:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829537.639405:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787829537.639407:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829537/real 1787829537] req@ffff8d30c7014f00 x1874670646840064/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829562 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787829537.639411:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787829537.639412:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787829537.639413:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787829537.639414:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787829537.639415:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829537.639416:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829537.639416:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787829537.639419:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c7014f00 x1874670646840064/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829562 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787829537.639422:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829537.639423:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787829537.639425:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829537.639425:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829537.639426:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829537.639427:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787829537.639428:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787829537.639429:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829537.639431:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787829537.639433:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787829537.639434:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c7014f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646840064:192.168.203.143@tcp:8:kworker.0 00000100:00000001:1.0:1787829537.639436:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829537.639437:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829537.639438:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787829537.639439:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787829537.639440:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787829537.639441:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787829537.639442:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787829537.639443:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787829537.639443:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787829537.639445:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787829537.639445:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829537.639446:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829537.639448:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787829537.639449:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829537.639450:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787829537.639451:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829538.534669:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829538.534673:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829538.534716:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829538.534719:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829538.534721:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829539.558690:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829539.558701:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829539.558757:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829539.558761:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829539.558775:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829540.582718:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829540.582724:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829540.582781:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829540.582785:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829540.582788:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829541.606673:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829541.606679:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829541.606703:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829541.606706:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829541.606708:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829542.630670:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829542.630674:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829542.630716:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829542.630719:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829542.630722:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787829542.758709:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829542.758715:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1787829542.758717:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787829542.758720:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787829542.758723:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787829542.758729:0:468:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 4343 00000100:00080000:2.0:1787829542.758737:0:468:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787829542.758740:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829542.758741:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787829542.758745:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787829542.758747:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787829542.758749:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:2.0:1787829542.758750:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:2.0:1787829542.758751:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829542.758752:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:2.0:1787829542.758756:0:468:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787829542.758758:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829542.758760:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787829542.758762:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787829542.758764:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829542.758765:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:2.0:1787829542.758767:0:468:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829542.758774:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787829542.758775:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829542.758778:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787829542.758781:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787829542.758783:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829542.758792:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787829542.758799:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2ed00 x1874670646840192/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787829542.758806:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787829542.758810:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829542.758821:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787829542.758824:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829542.758827:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1787829542.758828:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787829542.758830:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787829542.758832:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787829542.758835:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4343 00000100:00080000:2.0:1787829542.758838:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787829542.758840:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829542.758841:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787829542.758843:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787829542.758845:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787829542.758847:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:3.0:1787829542.758847:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:2.0:1787829542.758849:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:2.0:1787829542.758852:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829542.758853:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:3.0:1787829542.758854:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00080000:2.0:1787829542.758856:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:3.0:1787829542.758859:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1787829542.758860:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829542.758863:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:3.0:1787829542.758864:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 02000000:00000001:2.0:1787829542.758868:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829542.758868:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829542.758870:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829542.758872:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1787829542.758875:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1787829542.758878:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829542.758880:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829542.758880:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787829542.758884:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787829542.758885:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787829542.758888:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829542.758888:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829542.758893:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787829542.758895:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787829542.758896:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829542.758899:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787829542.758903:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2f0c0 x1874670646840320/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787829542.758907:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787829542.758909:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829542.758910:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787829542.758912:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829542.758914:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1787829542.758914:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787829542.758915:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787829542.758917:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787829542.758919:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4343 00000100:00080000:2.0:1787829542.758920:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787829542.758922:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829542.758922:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787829542.758923:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787829542.758924:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787829542.758925:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000020:00000001:2.0:1787829542.758926:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:2.0:1787829542.758927:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829542.758928:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:2.0:1787829542.758930:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787829542.758931:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829542.758932:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787829542.758933:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787829542.758935:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829542.758936:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1787829542.758938:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1787829542.758939:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829542.758940:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829542.758943:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00100000:3.0:1787829542.758943:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646840192:192.168.203.143@tcp:250:kworker.0 02000000:00000001:2.0:1787829542.758945:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829542.758949:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:3.0:1787829542.758950:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787829542.758951:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787829542.758954:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829542.758956:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829542.758956:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1787829542.758959:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787829542.758961:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2c780 x1874670646840448/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:3.0:1787829542.758963:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:2.0:1787829542.758968:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 02000000:00000001:3.0:1787829542.758968:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829542.758970:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829542.758973:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787829542.758974:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829542.758976:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1787829542.758978:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787829542.758979:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787829542.758980:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787829542.758982:0:468:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4343 00000100:00000001:3.0:1787829542.758982:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787829542.758985:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417879808 : -126234291671808 : ffff8d30c9e2ed00) 00000100:00080000:2.0:1787829542.758987:0:468:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787829542.758990:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829542.758990:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:2.0:1787829542.758991:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787829542.758993:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787829542.758994:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787829542.758995:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000020:00000001:2.0:1787829542.758996:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:2.0:1787829542.758998:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829542.758998:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:2.0:1787829542.759000:0:468:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787829542.759002:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829542.759003:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787829542.759004:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787829542.759005:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829542.759006:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1787829542.759007:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000008:00000001:2.0:1787829542.759008:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829542.759009:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829542.759011:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787829542.759012:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829542.759013:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787829542.759013:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787829542.759014:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829542.759017:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787829542.759019:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2fc00 x1874670646840576/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787829542.759023:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787829542.759024:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829542.759025:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:3.0:1787829542.759143:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:3.0:1787829542.759153:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787829542.759156:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829542.759158:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829542.759159:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1787829542.759164:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829542.759167:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4299016960 00000100:00000001:3.0:1787829542.759175:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 00000100:00000001:3.0:1787829542.759179:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4299016960 00000100:00000001:3.0:1787829542.759183:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 00000100:00000001:3.0:1787829542.759186:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829542.759189:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829542.759191:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1787829542.759197:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829542/real 1787829542] req@ffff8d30c9e2ed00 x1874670646840192/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829567 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1787829542.759207:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787829542.759209:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829542.759212:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829542.759214:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829542.759215:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829542.759217:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829542.759218:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829542.759221:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2ed00 x1874670646840192/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829567 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829542.759226:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829542.759228:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829542.759230:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829542.759231:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829542.759232:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829542.759233:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829542.759234:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829542.759236:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829542.759238:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829542.759240:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829542.759242:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646840192:192.168.203.143@tcp:250:kworker.0 00000100:00000001:3.0:1787829542.759246:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829542.759247:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829542.759248:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829542.759250:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829542.759252:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829542.759253:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829542.759254:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829542.759256:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829542.759257:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829542.759261:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829542.759261:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829542.759263:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829542.759268:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787829542.759269:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829542.759270:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1787829542.759271:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787829542.759272:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787829542.759273:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787829542.759274:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829542.759275:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787829542.759277:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829542.759278:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829542.759280:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646840320:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829542.759282:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1787829542.759283:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1787829542.759284:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829542.759285:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1787829542.759287:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829542.759290:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787829542.759291:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417880768 : -126234291670848 : ffff8d30c9e2f0c0) 00000100:00000001:3.0:1787829542.759295:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1787829542.759303:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:3.0:1787829542.759306:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787829542.759307:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829542.759308:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829542.759309:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1787829542.759311:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829542.759314:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4299015702 00000100:00000001:3.0:1787829542.759316:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4299016960 00000100:00000001:3.0:1787829542.759317:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4299015702 00000100:00000001:3.0:1787829542.759318:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4299016960 00000100:00000001:3.0:1787829542.759319:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829542.759320:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829542.759321:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1787829542.759336:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829542/real 1787829542] req@ffff8d30c9e2f0c0 x1874670646840320/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829567 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1787829542.759341:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787829542.759342:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829542.759344:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829542.759345:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829542.759346:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829542.759347:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829542.759348:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829542.759350:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2f0c0 x1874670646840320/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829567 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829542.759355:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829542.759356:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829542.759357:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829542.759358:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829542.759359:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829542.759360:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829542.759361:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829542.759362:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829542.759364:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829542.759365:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829542.759366:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646840320:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829542.759369:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787829542.759370:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787829542.759371:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829542.759372:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787829542.759373:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829542.759374:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829542.759376:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646840448:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829542.759378:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1787829542.759379:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1787829542.759380:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829542.759381:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1787829542.759382:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829542.759385:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787829542.759386:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417870208 : -126234291681408 : ffff8d30c9e2c780) 00000100:00000001:3.0:1787829542.759388:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1787829542.759396:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:3.0:1787829542.759398:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787829542.759400:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829542.759400:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829542.759402:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1787829542.759403:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829542.759405:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4299015702 00000100:00000001:3.0:1787829542.759407:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4299016960 00000100:00000001:3.0:1787829542.759408:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4299015702 00000100:00000001:3.0:1787829542.759409:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4299016960 00000100:00000001:3.0:1787829542.759411:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829542.759412:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829542.759413:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1787829542.759415:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829542/real 1787829542] req@ffff8d30c9e2c780 x1874670646840448/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829567 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1787829542.759419:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787829542.759419:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829542.759421:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829542.759422:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829542.759422:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829542.759423:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829542.759424:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829542.759426:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2c780 x1874670646840448/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829567 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829542.759429:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829542.759431:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829542.759432:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829542.759433:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829542.759433:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829542.759434:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829542.759435:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829542.759436:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829542.759438:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829542.759439:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829542.759441:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646840448:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829542.759443:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787829542.759444:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787829542.759445:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829542.759446:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787829542.759447:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829542.759448:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829542.759449:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646840576:192.168.203.143@tcp:8:kworker.0 00000100:00000001:3.0:1787829542.759450:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1787829542.759451:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1787829542.759452:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829542.759453:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1787829542.759454:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829542.759457:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787829542.759458:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417883648 : -126234291667968 : ffff8d30c9e2fc00) 00000100:00000001:3.0:1787829542.759459:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1787829542.759464:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:3.0:1787829542.759466:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787829542.759467:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829542.759468:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829542.759469:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1787829542.759470:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829542.759472:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4299015715 00000100:00000001:3.0:1787829542.759473:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4299016960 00000100:00000001:3.0:1787829542.759474:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4299015715 00000100:00000001:3.0:1787829542.759475:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4299016960 00000100:00000001:3.0:1787829542.759476:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829542.759477:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829542.759478:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1787829542.759480:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829542/real 1787829542] req@ffff8d30c9e2fc00 x1874670646840576/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829567 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1787829542.759488:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787829542.759489:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829542.759490:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829542.759491:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829542.759492:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829542.759493:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829542.759493:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829542.759495:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2fc00 x1874670646840576/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829567 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829542.759499:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829542.759500:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829542.759501:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829542.759502:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829542.759502:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829542.759504:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829542.759505:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829542.759506:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829542.759508:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829542.759509:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829542.759510:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646840576:192.168.203.143@tcp:8:kworker.0 00000100:00000001:3.0:1787829542.759512:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829542.759513:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829542.759514:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829542.759515:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829542.759516:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829542.759516:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829542.759517:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829542.759518:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829542.759519:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829542.759520:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829542.759521:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829542.759522:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829542.759523:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829542.759523:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829542.759524:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829542.759525:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829542.759526:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829542.759527:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829542.759527:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829542.759529:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829542.759529:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829542.759530:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829542.759531:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829542.759531:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829542.759532:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829542.759532:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829542.759533:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829542.759534:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829542.759534:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829542.759535:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829542.759536:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829542.759537:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829542.759538:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787829542.759539:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829542.759540:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1787829542.759541:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829543.654657:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829543.654662:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829543.654698:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829543.654701:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829543.654704:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829544.678700:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829544.678707:0:815:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1787829544.678709:0:815:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1787829544.678711:0:815:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829544.678715:0:815:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1787829544.678717:0:815:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829544.678718:0:815:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829544.678720:0:815:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829544.678784:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829544.678788:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829544.678791:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829545.702663:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829545.702670:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829545.702725:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829545.702729:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829545.702733:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829546.726693:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829546.726698:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829546.726757:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829546.726761:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829546.726763:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829547.750697:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829547.750704:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829547.750752:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829547.750755:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829547.750758:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787829547.878735:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829547.878746:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1787829547.878749:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787829547.878753:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787829547.878757:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787829547.878764:0:468:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 4348 00000100:00080000:3.0:1787829547.878776:0:468:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787829547.878779:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829547.878781:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:3.0:1787829547.878784:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1787829547.878786:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1787829547.878788:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:3.0:1787829547.878790:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:3.0:1787829547.878791:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829547.878792:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:3.0:1787829547.878795:0:468:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:3.0:1787829547.878798:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829547.878801:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787829547.878804:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829547.878807:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829547.878810:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:3.0:1787829547.878813:0:468:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829547.878824:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787829547.878827:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829547.878839:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787829547.878842:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787829547.878847:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829547.878856:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829547.878863:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c70143c0 x1874670646840704/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787829547.878872:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787829547.878875:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829547.878888:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787829547.878890:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829547.878892:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1787829547.878893:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787829547.878895:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787829547.878897:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787829547.878898:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4348 00000100:00080000:3.0:1787829547.878901:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787829547.878903:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829547.878905:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:3.0:1787829547.878907:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1787829547.878908:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1787829547.878910:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000020:00000001:3.0:1787829547.878912:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:3.0:1787829547.878913:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829547.878919:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:1.0:1787829547.878922:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00080000:3.0:1787829547.878923:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:3.0:1787829547.878928:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829547.878929:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1787829547.878930:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1787829547.878934:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 02000000:00000001:3.0:1787829547.878935:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829547.878939:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 02000000:00000001:3.0:1787829547.878939:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829547.878943:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:1.0:1787829547.878944:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000008:00100000:3.0:1787829547.878947:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 02000000:00000001:1.0:1787829547.878950:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000008:00000001:3.0:1787829547.878951:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829547.878955:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829547.878956:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829547.878958:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1787829547.878960:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829547.878961:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829547.878965:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787829547.878967:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00100000:1.0:1787829547.878969:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c70143c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646840704:192.168.203.143@tcp:250:kworker.0 02000000:00000001:3.0:1787829547.878969:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829547.878973:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829547.878979:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00100000:3.0:1787829547.878982:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c7015680 x1874670646840832/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:1.0:1787829547.878985:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787829547.878990:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829547.878992:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 02000000:00000001:1.0:1787829547.878994:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:3.0:1787829547.878995:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829547.878999:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787829547.879001:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 02000000:00000001:1.0:1787829547.879002:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787829547.879005:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1787829547.879008:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787829547.879009:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787829547.879011:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787829547.879014:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4348 00000100:00080000:3.0:1787829547.879016:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787829547.879018:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829547.879018:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:3.0:1787829547.879020:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:1.0:1787829547.879021:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000020:00000001:3.0:1787829547.879022:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:1.0:1787829547.879026:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369536448 : -126234340015168 : ffff8d30c70143c0) 00000020:00000001:3.0:1787829547.879032:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000020:00000001:3.0:1787829547.879034:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:3.0:1787829547.879036:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829547.879037:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:3.0:1787829547.879042:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:1.0:1787829547.879045:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:3.0:1787829547.879046:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829547.879050:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787829547.879052:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829547.879053:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829547.879055:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:3.0:1787829547.879056:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1787829547.879058:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829547.879059:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829547.879063:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787829547.879064:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829547.879065:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787829547.879066:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787829547.879068:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829547.879071:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829547.879073:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c70152c0 x1874670646840960/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787829547.879079:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787829547.879080:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829547.879082:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787829547.879084:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:1.0:1787829547.879091:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00080000:3.0:1787829547.879091:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1787829547.879095:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787829547.879098:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787829547.879100:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787829547.879103:0:468:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4348 00000100:00000001:1.0:1787829547.879105:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00080000:3.0:1787829547.879106:0:468:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787829547.879110:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829547.879111:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:3.0:1787829547.879113:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1787829547.879114:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1787829547.879115:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000020:00000001:3.0:1787829547.879116:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:3.0:1787829547.879118:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829547.879118:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:3.0:1787829547.879120:0:468:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:1.0:1787829547.879121:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829547.879122:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829547.879125:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1787829547.879126:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829547.879127:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829547.879132:0:2390:0:(events.c:68:request_out_callback()) Process leaving 02000000:00000001:3.0:1787829547.879132:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829547.879136:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:3.0:1787829547.879138:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1787829547.879140:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829547.879141:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829547.879142:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829547.879144:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787829547.879147:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829547.879149:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:1.0:1787829547.879150:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4299022080 02000000:00000001:3.0:1787829547.879150:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787829547.879152:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829547.879158:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829547.879161:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c7014b40 x1874670646841088/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787829547.879165:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 00000100:00000001:3.0:1787829547.879168:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787829547.879171:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829547.879173:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829547.879183:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4299022080 00000100:00000001:1.0:1787829547.879191:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 00000100:00000001:1.0:1787829547.879196:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829547.879200:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829547.879203:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787829547.879210:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829547/real 1787829547] req@ffff8d30c70143c0 x1874670646840704/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829572 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787829547.879222:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787829547.879225:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787829547.879229:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787829547.879232:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787829547.879234:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829547.879237:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829547.879240:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787829547.879245:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c70143c0 x1874670646840704/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829572 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787829547.879253:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829547.879256:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787829547.879259:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829547.879261:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829547.879262:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829547.879265:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787829547.879267:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787829547.879270:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829547.879274:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787829547.879278:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787829547.879281:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c70143c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646840704:192.168.203.143@tcp:250:kworker.0 00000100:00000001:1.0:1787829547.879286:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829547.879301:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829547.879303:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787829547.879305:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787829547.879308:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787829547.879310:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787829547.879312:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787829547.879317:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787829547.879319:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787829547.879321:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787829547.879322:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829547.879325:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829547.879331:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787829547.879333:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829547.879336:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787829547.879339:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1787829547.879340:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787829547.879342:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787829547.879344:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829547.879347:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787829547.879349:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829547.879352:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829547.879355:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c7015680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646840832:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787829547.879359:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787829547.879361:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787829547.879362:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829547.879364:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787829547.879367:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829547.879373:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787829547.879375:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369541248 : -126234340010368 : ffff8d30c7015680) 00000100:00000001:1.0:1787829547.879379:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1787829547.879394:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787829547.879399:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787829547.879401:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829547.879403:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829547.879405:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787829547.879408:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829547.879412:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4299020822 00000100:00000001:1.0:1787829547.879415:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4299022080 00000100:00000001:1.0:1787829547.879418:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4299020822 00000100:00000001:1.0:1787829547.879420:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4299022080 00000100:00000001:1.0:1787829547.879422:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829547.879425:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829547.879426:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787829547.879431:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829547/real 1787829547] req@ffff8d30c7015680 x1874670646840832/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829572 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787829547.879441:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787829547.879442:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787829547.879446:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787829547.879447:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787829547.879449:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829547.879451:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829547.879452:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787829547.879457:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c7015680 x1874670646840832/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829572 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787829547.879465:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829547.879467:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787829547.879470:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829547.879472:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829547.879474:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829547.879475:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787829547.879477:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787829547.879480:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829547.879483:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787829547.879485:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787829547.879488:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c7015680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646840832:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787829547.879492:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787829547.879493:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787829547.879495:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829547.879497:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787829547.879499:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829547.879501:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829547.879504:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c70152c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646840960:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787829547.879508:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787829547.879510:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787829547.879512:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829547.879513:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787829547.879516:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829547.879521:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787829547.879523:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369540288 : -126234340011328 : ffff8d30c70152c0) 00000100:00000001:1.0:1787829547.879526:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1787829547.879538:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787829547.879543:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787829547.879545:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829547.879547:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829547.879549:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787829547.879552:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829547.879556:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4299020822 00000100:00000001:1.0:1787829547.879559:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4299022080 00000100:00000001:1.0:1787829547.879561:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4299020822 00000100:00000001:1.0:1787829547.879563:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4299022080 00000100:00000001:1.0:1787829547.879565:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829547.879567:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829547.879569:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787829547.879573:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829547/real 1787829547] req@ffff8d30c70152c0 x1874670646840960/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829572 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787829547.879582:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787829547.879584:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787829547.879586:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787829547.879588:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787829547.879590:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829547.879592:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829547.879593:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787829547.879597:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c70152c0 x1874670646840960/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829572 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787829547.879626:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829547.879629:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787829547.879631:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829547.879633:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829547.879634:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829547.879636:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787829547.879638:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787829547.879640:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829547.879642:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787829547.879644:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787829547.879648:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c70152c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646840960:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787829547.879652:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787829547.879654:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787829547.879655:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829547.879657:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787829547.879659:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829547.879661:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829547.879665:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c7014b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646841088:192.168.203.143@tcp:8:kworker.0 00000100:00000001:1.0:1787829547.879668:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787829547.879670:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787829547.879672:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829547.879673:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787829547.879676:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829547.879682:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787829547.879684:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369538368 : -126234340013248 : ffff8d30c7014b40) 00000100:00000001:1.0:1787829547.879688:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1787829547.879701:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787829547.879706:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787829547.879708:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829547.879710:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829547.879711:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787829547.879714:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829547.879718:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4299020836 00000100:00000001:1.0:1787829547.879720:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4299022080 00000100:00000001:1.0:1787829547.879722:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4299020836 00000100:00000001:1.0:1787829547.879724:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4299022080 00000100:00000001:1.0:1787829547.879726:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829547.879728:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829547.879730:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787829547.879735:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829547/real 1787829547] req@ffff8d30c7014b40 x1874670646841088/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829572 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787829547.879748:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787829547.879750:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787829547.879753:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787829547.879754:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787829547.879755:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829547.879758:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829547.879759:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787829547.879763:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c7014b40 x1874670646841088/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829572 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787829547.879771:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829547.879773:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787829547.879776:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829547.879777:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829547.879778:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829547.879780:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787829547.879782:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787829547.879785:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829547.879788:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787829547.879790:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787829547.879794:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c7014b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646841088:192.168.203.143@tcp:8:kworker.0 00000100:00000001:1.0:1787829547.879797:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829547.879800:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829547.879801:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787829547.879803:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787829547.879805:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787829547.879807:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787829547.879808:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787829547.879810:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787829547.879812:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787829547.879814:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787829547.879816:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829547.879818:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829547.879819:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787829547.879821:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787829547.879823:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787829547.879824:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787829547.879825:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787829547.879827:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787829547.879828:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787829547.879829:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787829547.879831:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829547.879833:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829547.879834:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787829547.879835:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787829547.879837:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787829547.879838:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787829547.879839:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787829547.879841:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787829547.879842:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787829547.879843:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787829547.879845:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829547.879847:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829547.879850:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787829547.879852:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829547.879853:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787829547.879855:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829548.774711:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829548.774719:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829548.774760:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829548.774763:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829548.774766:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829549.798684:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829549.798689:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829549.798736:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829549.798739:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829549.798741:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829550.822675:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829550.822680:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829550.822737:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829550.822741:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829550.822745:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829551.846675:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829551.846681:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829551.846738:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829551.846741:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829551.846744:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829552.870695:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829552.870701:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829552.870780:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829552.870783:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829552.870786:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787829552.998732:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829552.998744:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1787829552.998747:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787829552.998750:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787829552.998755:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787829552.998762:0:468:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 4353 00000100:00080000:0.0:1787829552.998774:0:468:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787829552.998777:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829552.998779:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787829552.998783:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787829552.998786:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787829552.998789:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:0.0:1787829552.998792:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:0.0:1787829552.998794:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829552.998796:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:0.0:1787829552.998801:0:468:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787829552.998804:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829552.998807:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787829552.998811:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787829552.998814:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829552.998816:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1787829552.998819:0:468:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829552.998829:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787829552.998832:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829552.998835:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787829552.998838:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787829552.998842:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829552.998851:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787829552.998859:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d25a40 x1874670646841216/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787829552.998866:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787829552.998870:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829552.998885:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787829552.998887:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829552.998891:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1787829552.998892:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787829552.998893:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787829552.998896:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787829552.998897:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4353 00000100:00080000:0.0:1787829552.998900:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787829552.998901:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829552.998902:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787829552.998904:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787829552.998905:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787829552.998907:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000020:00000001:0.0:1787829552.998908:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:0.0:1787829552.998910:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829552.998911:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:0.0:1787829552.998914:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787829552.998915:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829552.998917:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787829552.998921:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787829552.998922:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829552.998924:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:1.0:1787829552.998926:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000008:00100000:0.0:1787829552.998928:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1787829552.998930:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829552.998932:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829552.998934:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787829552.998935:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829552.998936:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787829552.998939:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787829552.998941:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829552.998942:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787829552.998945:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829552.998945:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787829552.998948:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787829552.998951:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787829552.998952:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d24f00 x1874670646841344/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:1.0:1787829552.998955:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787829552.998960:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829552.998965:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829552.998973:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d25a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646841216:192.168.203.143@tcp:250:kworker.0 00000100:00000001:0.0:1787829552.998975:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787829552.998978:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:0.0:1787829552.998980:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829552.998983:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:0.0:1787829552.998987:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829552.998987:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829552.998990:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787829552.998994:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787829552.999001:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829552.999006:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:1.0:1787829552.999006:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787829552.999010:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601343040 : -126234108208576 : ffff8d30d4d25a40) 00000100:00000001:0.0:1787829552.999011:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:1.0:1787829552.999016:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00080000:0.0:1787829552.999017:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787829552.999022:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787829552.999026:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4353 00000100:00080000:0.0:1787829552.999040:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787829552.999043:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829552.999046:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787829552.999049:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787829552.999051:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787829552.999052:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000020:00000001:0.0:1787829552.999057:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000800:00000100:1.0:1787829552.999059:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:0.0:1787829552.999061:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829552.999064:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:0.0:1787829552.999070:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787829552.999073:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829552.999075:0:2390:0:(events.c:34:request_out_callback()) Process entered 02000000:00000001:0.0:1787829552.999076:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1787829552.999081:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 02000000:00000001:0.0:1787829552.999082:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787829552.999087:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829552.999087:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829552.999090:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:1.0:1787829552.999093:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000008:00100000:0.0:1787829552.999094:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1787829552.999098:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829552.999099:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829552.999101:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829552.999104:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787829552.999107:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829552.999107:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4299027200 00000100:00000001:0.0:1787829552.999113:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787829552.999115:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787829552.999116:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829552.999120:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1787829552.999121:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787829552.999128:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d24000 x1874670646841472/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787829552.999128:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4299027200 00000100:00000001:1.0:1787829552.999136:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1787829552.999140:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787829552.999141:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787829552.999145:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829552.999146:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829552.999151:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00000001:0.0:1787829552.999152:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787829552.999157:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1787829552.999158:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829552/real 1787829552] req@ffff8d30d4d25a40 x1874670646841216/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829577 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829552.999168:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1787829552.999172:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000400:00000001:1.0:1787829552.999175:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00080000:0.0:1787829552.999177:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000400:00000001:1.0:1787829552.999178:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787829552.999183:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787829552.999184:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:1.0:1787829552.999186:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787829552.999189:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:0.0:1787829552.999192:0:468:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4353 00000100:00000001:1.0:1787829552.999193:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829552.999196:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:0.0:1787829552.999202:0:468:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1787829552.999203:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d25a40 x1874670646841216/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829577 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787829552.999208:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829552.999212:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787829552.999219:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787829552.999222:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:1.0:1787829552.999223:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1787829552.999227:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:1.0:1787829552.999228:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000020:00000001:0.0:1787829552.999232:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:1.0:1787829552.999233:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829552.999237:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787829552.999237:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787829552.999241:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:1.0:1787829552.999241:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829552.999248:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00080000:0.0:1787829552.999250:0:468:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:1.0:1787829552.999251:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787829552.999269:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00000001:0.0:1787829552.999270:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829552.999273:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00080000:1.0:1787829552.999276:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 02000000:00000001:0.0:1787829552.999278:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829552.999279:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 02000000:00000001:0.0:1787829552.999284:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829552.999287:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d25a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646841216:192.168.203.143@tcp:250:kworker.0 00000100:00000001:0.0:1787829552.999288:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1787829552.999292:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1787829552.999296:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829552.999297:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829552.999298:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:0.0:1787829552.999301:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:1.0:1787829552.999302:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 02000000:00000001:0.0:1787829552.999304:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829552.999305:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 00000100:00000001:0.0:1787829552.999307:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1787829552.999309:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787829552.999310:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787829552.999313:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829552.999314:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00000001:0.0:1787829552.999318:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829552.999318:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787829552.999321:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 00000100:00100000:0.0:1787829552.999326:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d270c0 x1874670646841600/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:1.0:1787829552.999326:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787829552.999328:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787829552.999332:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787829552.999335:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829552.999336:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787829552.999338:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829552.999340:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829552.999343:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829552.999346:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787829552.999349:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829552.999351:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787829552.999352:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1787829552.999354:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787829552.999355:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787829552.999356:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829552.999357:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787829552.999359:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829552.999361:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829552.999365:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d24f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646841344:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787829552.999368:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787829552.999369:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787829552.999371:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829552.999372:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787829552.999374:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829552.999379:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787829552.999380:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601340160 : -126234108211456 : ffff8d30d4d24f00) 00000100:00000001:1.0:1787829552.999384:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1787829552.999396:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787829552.999401:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787829552.999404:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829552.999405:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829552.999407:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787829552.999410:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829552.999414:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4299025942 00000100:00000001:1.0:1787829552.999418:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4299027200 00000100:00000001:1.0:1787829552.999420:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4299025942 00000100:00000001:1.0:1787829552.999422:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4299027200 00000100:00000001:1.0:1787829552.999424:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829552.999426:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829552.999429:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787829552.999434:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829552/real 1787829552] req@ffff8d30d4d24f00 x1874670646841344/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829577 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787829552.999443:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787829552.999444:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787829552.999448:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787829552.999449:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787829552.999451:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829552.999453:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829552.999455:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787829552.999459:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d24f00 x1874670646841344/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829577 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787829552.999474:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829552.999476:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787829552.999479:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829552.999481:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829552.999482:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829552.999484:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787829552.999486:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787829552.999489:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829552.999492:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787829552.999494:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787829552.999497:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d24f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646841344:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787829552.999500:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787829552.999501:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787829552.999502:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829552.999504:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787829552.999505:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829552.999507:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829552.999509:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d24000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646841472:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787829552.999511:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787829552.999512:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787829552.999514:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829552.999515:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787829552.999517:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829552.999521:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787829552.999522:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601336320 : -126234108215296 : ffff8d30d4d24000) 00000100:00000001:1.0:1787829552.999525:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1787829552.999534:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787829552.999538:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787829552.999539:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829552.999540:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829552.999541:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787829552.999543:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829552.999546:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4299025942 00000100:00000001:1.0:1787829552.999547:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4299027200 00000100:00000001:1.0:1787829552.999549:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4299025942 00000100:00000001:1.0:1787829552.999551:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4299027200 00000100:00000001:1.0:1787829552.999552:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829552.999553:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829552.999554:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787829552.999557:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829552/real 1787829552] req@ffff8d30d4d24000 x1874670646841472/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829577 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787829552.999562:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787829552.999562:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787829552.999564:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787829552.999565:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787829552.999566:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829552.999567:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829552.999568:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787829552.999570:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d24000 x1874670646841472/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829577 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787829552.999575:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829552.999577:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787829552.999578:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829552.999579:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829552.999580:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829552.999581:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787829552.999582:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787829552.999583:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829552.999585:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787829552.999586:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787829552.999589:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d24000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646841472:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787829552.999591:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787829552.999592:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787829552.999593:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829552.999595:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787829552.999596:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829552.999597:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829552.999599:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d270c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646841600:192.168.203.143@tcp:8:kworker.0 00000100:00000001:1.0:1787829552.999622:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787829552.999624:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787829552.999624:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829552.999626:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787829552.999628:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829552.999631:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787829552.999632:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601348800 : -126234108202816 : ffff8d30d4d270c0) 00000100:00000001:1.0:1787829552.999635:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1787829552.999642:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787829552.999645:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787829552.999647:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829552.999648:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829552.999649:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787829552.999651:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829552.999653:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4299025956 00000100:00000001:1.0:1787829552.999655:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4299027200 00000100:00000001:1.0:1787829552.999656:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4299025956 00000100:00000001:1.0:1787829552.999657:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4299027200 00000100:00000001:1.0:1787829552.999659:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829552.999660:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829552.999661:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787829552.999664:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829552/real 1787829552] req@ffff8d30d4d270c0 x1874670646841600/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829577 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787829552.999673:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787829552.999674:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787829552.999675:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787829552.999676:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787829552.999677:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829552.999678:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829552.999679:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787829552.999681:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d270c0 x1874670646841600/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829577 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787829552.999686:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829552.999688:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787829552.999689:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829552.999690:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829552.999691:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829552.999692:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787829552.999693:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787829552.999694:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829552.999696:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787829552.999697:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787829552.999699:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d270c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646841600:192.168.203.143@tcp:8:kworker.0 00000100:00000001:1.0:1787829552.999701:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829552.999702:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829552.999703:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787829552.999704:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787829552.999705:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787829552.999706:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787829552.999707:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787829552.999709:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787829552.999710:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787829552.999712:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787829552.999713:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829552.999715:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829552.999715:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787829552.999716:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787829552.999718:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787829552.999718:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787829552.999719:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787829552.999720:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787829552.999721:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787829552.999722:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787829552.999723:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829552.999724:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829552.999724:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787829552.999725:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787829552.999726:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787829552.999727:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787829552.999727:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787829552.999728:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787829552.999729:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787829552.999731:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787829552.999732:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829552.999733:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829552.999735:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787829552.999736:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829552.999737:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787829552.999738:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829553.894687:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829553.894695:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829553.894762:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829553.894767:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829553.894771:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829554.918704:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829554.918711:0:815:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1787829554.918712:0:815:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1787829554.918715:0:815:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829554.918719:0:815:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1787829554.918721:0:815:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829554.918722:0:815:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829554.918724:0:815:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829554.918791:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829554.918796:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829554.918799:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829555.942688:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829555.942693:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829555.942751:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829555.942756:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829555.942759:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829556.966675:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829556.966681:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829556.966737:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829556.966742:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829556.966746:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829557.990687:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829557.990693:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829557.990742:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829557.990745:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829557.990747:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787829558.118693:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829558.118705:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1787829558.118707:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787829558.118710:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787829558.118714:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787829558.118721:0:40798:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 4358 00000100:00080000:2.0:1787829558.118729:0:40798:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787829558.118731:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829558.118733:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787829558.118736:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787829558.118739:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787829558.118742:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:2.0:1787829558.118744:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:2.0:1787829558.118745:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829558.118746:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:2.0:1787829558.118749:0:40798:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787829558.118751:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829558.118753:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787829558.118755:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787829558.118757:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829558.118759:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:2.0:1787829558.118761:0:40798:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829558.118770:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787829558.118771:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829558.118774:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787829558.118776:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787829558.118779:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829558.118788:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787829558.118795:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2de00 x1874670646841728/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787829558.118803:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787829558.118805:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829558.118817:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787829558.118819:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829558.118822:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1787829558.118823:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787829558.118825:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787829558.118828:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787829558.118830:0:40798:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4358 00000100:00080000:2.0:1787829558.118834:0:40798:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787829558.118836:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829558.118838:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787829558.118841:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787829558.118843:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787829558.118846:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000020:00000001:2.0:1787829558.118848:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:2.0:1787829558.118851:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829558.118853:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:2.0:1787829558.118857:0:40798:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:1.0:1787829558.118859:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787829558.118860:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829558.118864:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1787829558.118865:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 02000000:00000001:2.0:1787829558.118870:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829558.118871:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787829558.118875:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 02000000:00000001:2.0:1787829558.118876:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829558.118879:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:1.0:1787829558.118880:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000008:00100000:2.0:1787829558.118883:0:40798:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 02000000:00000001:1.0:1787829558.118886:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000008:00000001:2.0:1787829558.118887:0:40798:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829558.118891:0:40798:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829558.118893:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787829558.118895:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1787829558.118898:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829558.118898:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829558.118904:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787829558.118905:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787829558.118909:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829558.118910:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646841728:192.168.203.143@tcp:250:kworker.0 00000100:00000001:1.0:1787829558.118918:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:2.0:1787829558.118918:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829558.118924:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787829558.118929:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787829558.118929:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2c3c0 x1874670646841856/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:1.0:1787829558.118935:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787829558.118943:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829558.118943:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787829558.118949:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829558.118953:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787829558.118956:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829558.118958:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1787829558.118959:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:1.0:1787829558.118961:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00080000:2.0:1787829558.118961:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787829558.118966:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417875968 : -126234291675648 : ffff8d30c9e2de00) 00000100:00000001:2.0:1787829558.118967:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787829558.118971:0:40798:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4358 00000100:00000001:1.0:1787829558.118975:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00080000:2.0:1787829558.118976:0:40798:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787829558.118980:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829558.118981:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787829558.118983:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787829558.118984:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787829558.118986:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000020:00000001:2.0:1787829558.118988:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:2.0:1787829558.118989:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829558.118990:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:2.0:1787829558.118993:0:40798:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787829558.118995:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829558.118996:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787829558.118998:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787829558.118999:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829558.119001:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1787829558.119003:0:40798:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1787829558.119004:0:40798:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829558.119006:0:40798:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829558.119009:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787829558.119011:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829558.119012:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787829558.119013:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787829558.119015:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829558.119018:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787829558.119022:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2d2c0 x1874670646841984/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787829558.119026:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829558.119028:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787829558.119031:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829558.119033:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4299032320 00000100:00000001:2.0:1787829558.119034:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787829558.119037:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829558.119041:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1787829558.119042:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787829558.119044:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787829558.119046:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:1.0:1787829558.119048:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 00000100:00080000:2.0:1787829558.119048:0:40798:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4358 00000100:00080000:2.0:1787829558.119053:0:40798:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787829558.119056:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829558.119058:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787829558.119059:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787829558.119060:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787829558.119061:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000020:00000001:2.0:1787829558.119062:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:1.0:1787829558.119064:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4299032320 00000100:00000001:2.0:1787829558.119064:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829558.119067:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:2.0:1787829558.119071:0:40798:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:1.0:1787829558.119074:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 00000100:00000001:2.0:1787829558.119079:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829558.119081:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787829558.119082:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787829558.119085:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829558.119088:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829558.119088:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829558.119094:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:1.0:1787829558.119097:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000008:00100000:2.0:1787829558.119097:0:40798:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000008:00000001:2.0:1787829558.119103:0:40798:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829558.119105:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829558.119105:0:40798:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829558.119112:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787829558.119114:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829558.119115:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787829558.119116:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:1.0:1787829558.119117:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 02000000:00000001:2.0:1787829558.119118:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829558.119122:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:2.0:1787829558.119123:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829558.119130:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00100000:2.0:1787829558.119132:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2cb40 x1874670646842112/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787829558.119135:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1787829558.119140:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1787829558.119142:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787829558.119143:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787829558.119144:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829558.119148:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829558.119150:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829558.119155:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787829558.119160:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829558.119163:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829558.119169:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646841856:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787829558.119173:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787829558.119177:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787829558.119179:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829558.119181:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787829558.119185:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829558.119193:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787829558.119195:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417869248 : -126234291682368 : ffff8d30c9e2c3c0) 00000100:00000001:1.0:1787829558.119200:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:1.0:1787829558.119213:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829558.119231:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4299031062 00000100:00000001:1.0:1787829558.119234:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4299032320 00000100:00000001:1.0:1787829558.119236:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4299031062 00000100:00000001:1.0:1787829558.119239:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4299032320 00000100:00000001:1.0:1787829558.119241:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829558.119243:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829558.119246:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787829558.119247:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787829558.119248:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829558.119251:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787829558.119252:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829558.119255:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829558.119258:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2d2c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646841984:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787829558.119262:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787829558.119263:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787829558.119265:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829558.119267:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787829558.119270:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829558.119275:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787829558.119277:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417873088 : -126234291678528 : ffff8d30c9e2d2c0) 00000100:00000001:1.0:1787829558.119280:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:1.0:1787829558.119291:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829558.119296:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4299031062 00000100:00000001:1.0:1787829558.119298:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4299032320 00000100:00000001:1.0:1787829558.119301:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4299031062 00000100:00000001:1.0:1787829558.119303:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4299032320 00000100:00000001:1.0:1787829558.119305:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829558.119307:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829558.119310:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829558.119311:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829558.119315:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787829558.119316:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:1.0:1787829558.119318:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787829558.119319:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1787829558.119320:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787829558.119322:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787829558.119323:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829558.119325:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787829558.119327:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829558.119329:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829558.119333:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646842112:192.168.203.143@tcp:8:kworker.0 00000100:00000001:1.0:1787829558.119336:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787829558.119338:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787829558.119340:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829558.119341:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787829558.119344:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829558.119361:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787829558.119362:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417871168 : -126234291680448 : ffff8d30c9e2cb40) 00000100:00000001:1.0:1787829558.119366:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:1.0:1787829558.119376:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829558.119380:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4299031075 00000100:00000001:1.0:1787829558.119383:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4299032320 00000100:00000001:1.0:1787829558.119385:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4299031075 00000100:00000001:1.0:1787829558.119387:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4299032320 00000100:00000001:1.0:1787829558.119389:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829558.119391:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829558.119393:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829558.119395:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829558.119399:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787829558.119400:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:1.0:1787829558.119402:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787829558.119404:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1787829558.119406:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829558.119407:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829559.014695:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829559.014701:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829559.014751:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829559.014755:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829559.014758:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829560.038716:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829560.038721:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829560.038781:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829560.038784:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829560.038787:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829561.062668:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829561.062675:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829561.062722:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829561.062726:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829561.062728:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:1.0:1787829561.190852:0:2378:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.143:988 00000400:00000100:1.0:1787829561.190892:0:2378:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.143@tcp at host 192.168.203.143 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1787829561.190909:0:2378:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.143@tcp 00000100:00000001:1.0:1787829561.190922:0:2378:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787829561.190943:0:2378:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829561.190946:0:2378:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829561.190948:0:2378:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:1.0:1787829561.190959:0:2378:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.143@tcp 00000100:00000001:1.0:1787829561.190964:0:2378:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787829561.190967:0:2378:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829561.190968:0:2378:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829561.190970:0:2378:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:1.0:1787829561.190974:0:2378:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.143@tcp 00000100:00000001:1.0:1787829561.190977:0:2378:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787829561.190980:0:2378:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829561.190982:0:2378:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829561.190983:0:2378:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:1.0:1787829561.190988:0:2378:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.143@tcp 00000100:00000001:1.0:1787829561.190992:0:2378:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787829561.190995:0:2378:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829561.190996:0:2378:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829561.190997:0:2378:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1787829561.191083:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1787829561.191090:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787829561.191092:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1787829561.191104:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829558/real 1787829561] req@ffff8d30c9e2cb40 x1874670646842112/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829583 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1787829561.191120:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787829561.191123:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829561.191129:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829561.191132:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829561.191134:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829561.191138:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829561.191140:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829561.191145:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2cb40 x1874670646842112/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829583 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829561.191153:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829561.191157:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829561.191160:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829561.191162:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829561.191164:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829561.191166:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829561.191169:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829561.191172:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829561.191177:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829561.191180:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829561.191184:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646842112:192.168.203.143@tcp:8:kworker.0 00000100:00000001:3.0:1787829561.191189:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1787829561.191193:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829558/real 1787829561] req@ffff8d30c9e2c3c0 x1874670646841856/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829583 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1787829561.191215:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787829561.191216:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829561.191219:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829561.191221:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829561.191222:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829561.191224:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829561.191225:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829561.191230:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2c3c0 x1874670646841856/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829583 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829561.191236:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829561.191239:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829561.191241:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829561.191243:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829561.191244:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829561.191245:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829561.191247:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829561.191249:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829561.191253:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829561.191255:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829561.191259:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646841856:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829561.191263:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1787829561.191267:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829558/real 1787829561] req@ffff8d30c9e2d2c0 x1874670646841984/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829583 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1787829561.191276:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787829561.191277:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829561.191279:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829561.191281:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829561.191282:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829561.191284:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829561.191285:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829561.191289:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2d2c0 x1874670646841984/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829583 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829561.191296:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829561.191298:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829561.191300:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829561.191302:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829561.191303:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829561.191305:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829561.191307:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829561.191310:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829561.191314:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829561.191316:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829561.191319:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2d2c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646841984:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829561.191323:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1787829561.191327:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829558/real 1787829561] req@ffff8d30c9e2de00 x1874670646841728/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829583 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1787829561.191335:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787829561.191337:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829561.191340:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829561.191341:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829561.191343:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829561.191345:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829561.191346:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829561.191350:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2de00 x1874670646841728/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829583 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829561.191358:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829561.191360:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829561.191362:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829561.191363:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829561.191365:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829561.191366:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829561.191368:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829561.191370:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829561.191374:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829561.191376:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829561.191380:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646841728:192.168.203.143@tcp:250:kworker.0 00000100:00000001:3.0:1787829561.191385:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829561.191387:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829561.191389:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829561.191393:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829561.191398:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829561.191401:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829561.191404:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829561.191407:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829561.191409:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829561.191413:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829561.191414:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829561.191416:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829561.191417:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829561.191418:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829561.191421:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829561.191422:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829561.191423:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829561.191425:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829561.191427:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829561.191428:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829561.191429:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829561.191431:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829561.191432:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829561.191434:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829561.191435:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829561.191437:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829561.191438:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829561.191440:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829561.191441:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829561.191444:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829561.191445:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829561.191447:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829561.191448:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829561.191449:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829561.191451:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829561.191452:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829561.191454:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829561.191455:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829561.191456:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829561.191460:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829561.191462:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829561.191464:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829561.191472:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787829561.191475:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829561.191477:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1787829561.191479:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829562.086701:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829562.086706:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829562.086770:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829562.086773:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829562.086775:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829563.110671:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829563.110676:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829563.110707:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829563.110710:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829563.110713:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787829563.238680:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829563.238688:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1787829563.238690:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787829563.238693:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787829563.238697:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787829563.238702:0:40798:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 4363 00000100:00080000:2.0:1787829563.238710:0:40798:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787829563.238713:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829563.238715:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787829563.238718:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787829563.238721:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787829563.238723:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:2.0:1787829563.238725:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:2.0:1787829563.238727:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829563.238728:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:2.0:1787829563.238732:0:40798:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787829563.238733:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829563.238735:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787829563.238737:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787829563.238739:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829563.238740:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:2.0:1787829563.238742:0:40798:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829563.238751:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787829563.238752:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829563.238754:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787829563.238756:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787829563.238760:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829563.238770:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787829563.238780:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2d680 x1874670646842240/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787829563.238791:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787829563.238795:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829563.238813:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787829563.238816:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829563.238819:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1787829563.238821:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787829563.238823:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787829563.238825:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787829563.238828:0:40798:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4363 00000100:00080000:2.0:1787829563.238833:0:40798:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787829563.238835:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829563.238837:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787829563.238840:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787829563.238842:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787829563.238844:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:3.0:1787829563.238847:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:2.0:1787829563.238848:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:2.0:1787829563.238853:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829563.238854:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1787829563.238855:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:3.0:1787829563.238859:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00080000:2.0:1787829563.238863:0:40798:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:3.0:1787829563.238864:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787829563.238869:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829563.238869:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829563.238875:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787829563.238880:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787829563.238881:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829563.238885:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829563.238890:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00100000:3.0:1787829563.238893:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2d680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646842240:192.168.203.143@tcp:250:kworker.0 02000000:00000001:2.0:1787829563.238900:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829563.238902:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:2.0:1787829563.238907:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 02000000:00000001:3.0:1787829563.238909:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000008:00100000:2.0:1787829563.238912:0:40798:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 02000000:00000001:3.0:1787829563.238914:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:2.0:1787829563.238920:0:40798:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829563.238921:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:2.0:1787829563.238925:0:40798:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829563.238929:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829563.238933:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787829563.238936:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829563.238938:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787829563.238940:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787829563.238942:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829563.238948:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787829563.238949:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829563.238951:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417874048 : -126234291677568 : ffff8d30c9e2d680) 00000100:00000001:3.0:1787829563.238958:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00100000:2.0:1787829563.238960:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2e1c0 x1874670646842368/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787829563.238969:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787829563.238971:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829563.238975:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787829563.238978:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829563.238982:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1787829563.238984:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787829563.238985:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787829563.238989:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787829563.238992:0:40798:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4363 00000100:00080000:2.0:1787829563.238997:0:40798:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787829563.238999:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829563.239002:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000800:00000100:3.0:1787829563.239004:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000020:00000001:2.0:1787829563.239008:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787829563.239011:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787829563.239013:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000020:00000001:2.0:1787829563.239016:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:2.0:1787829563.239019:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829563.239020:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:3.0:1787829563.239022:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00080000:2.0:1787829563.239026:0:40798:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:3.0:1787829563.239029:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787829563.239034:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829563.239035:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829563.239039:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:3.0:1787829563.239040:0:2390:0:(events.c:68:request_out_callback()) Process leaving 02000000:00000001:2.0:1787829563.239045:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829563.239049:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829563.239050:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829563.239055:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:3.0:1787829563.239057:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4299037440 00000008:00100000:2.0:1787829563.239058:0:40798:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1787829563.239064:0:40798:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829563.239066:0:40798:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829563.239072:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:3.0:1787829563.239072:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 02000000:00000001:2.0:1787829563.239075:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829563.239078:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787829563.239080:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787829563.239082:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829563.239087:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787829563.239093:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2de00 x1874670646842496/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787829563.239100:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4299037440 00000100:00000001:2.0:1787829563.239101:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787829563.239106:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829563.239109:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 00000100:00000001:2.0:1787829563.239110:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787829563.239116:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1787829563.239117:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829563.239124:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787829563.239125:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1787829563.239129:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00000001:2.0:1787829563.239130:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787829563.239135:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787829563.239140:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00100000:3.0:1787829563.239141:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829563/real 1787829563] req@ffff8d30c9e2d680 x1874670646842240/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829588 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829563.239148:0:40798:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4363 00000100:00080000:2.0:1787829563.239155:0:40798:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787829563.239160:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000001:3.0:1787829563.239162:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000001:2.0:1787829563.239163:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000400:00000001:3.0:1787829563.239166:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000020:00000001:2.0:1787829563.239168:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787829563.239171:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:3.0:1787829563.239173:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000020:00000001:2.0:1787829563.239174:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:3.0:1787829563.239177:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000020:00000001:2.0:1787829563.239179:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:3.0:1787829563.239194:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829563.239197:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829563.239199:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787829563.239201:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:3.0:1787829563.239203:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:2.0:1787829563.239210:0:40798:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00100000:3.0:1787829563.239212:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2d680 x1874670646842240/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829588 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787829563.239217:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829563.239221:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787829563.239226:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787829563.239231:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829563.239236:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1787829563.239240:0:40798:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00000001:3.0:1787829563.239243:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000008:00000001:2.0:1787829563.239245:0:40798:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829563.239248:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787829563.239249:0:40798:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829563.239256:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829563.239258:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829563.239260:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829563.239262:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829563.239264:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829563.239267:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 02000000:00000001:2.0:1787829563.239273:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00080000:3.0:1787829563.239273:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 02000000:00000001:2.0:1787829563.239277:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829563.239278:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000001:2.0:1787829563.239281:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787829563.239286:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00100000:3.0:1787829563.239287:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2d680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646842240:192.168.203.143@tcp:250:kworker.0 02000000:00000001:2.0:1787829563.239290:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829563.239296:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829563.239299:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829563.239301:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829563.239305:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829563.239309:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 00000100:00100000:2.0:1787829563.239310:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2d2c0 x1874670646842624/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:3.0:1787829563.239314:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829563.239317:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829563.239321:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 00000100:00000001:2.0:1787829563.239324:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 02000000:00000001:3.0:1787829563.239327:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 00000100:00000001:2.0:1787829563.239328:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829563.239331:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1787829563.239333:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829563.239337:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829563.239340:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829563.239343:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829563.239351:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787829563.239353:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829563.239356:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1787829563.239358:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787829563.239360:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787829563.239361:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787829563.239363:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829563.239365:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787829563.239367:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829563.239369:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829563.239374:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646842368:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829563.239377:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1787829563.239379:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1787829563.239381:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829563.239384:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1787829563.239387:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829563.239393:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787829563.239395:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417876928 : -126234291674688 : ffff8d30c9e2e1c0) 00000100:00000001:3.0:1787829563.239400:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1787829563.239416:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:3.0:1787829563.239421:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787829563.239424:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829563.239425:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829563.239427:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1787829563.239430:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829563.239434:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4299036182 00000100:00000001:3.0:1787829563.239437:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4299037440 00000100:00000001:3.0:1787829563.239439:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4299036182 00000100:00000001:3.0:1787829563.239441:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4299037440 00000100:00000001:3.0:1787829563.239443:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829563.239445:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829563.239446:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1787829563.239450:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829563/real 1787829563] req@ffff8d30c9e2e1c0 x1874670646842368/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829588 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1787829563.239458:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787829563.239459:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829563.239462:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829563.239463:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829563.239465:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829563.239466:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829563.239468:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829563.239471:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2e1c0 x1874670646842368/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829588 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829563.239484:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829563.239486:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829563.239489:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829563.239490:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829563.239491:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829563.239493:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829563.239495:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829563.239497:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829563.239500:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829563.239503:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829563.239505:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646842368:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829563.239510:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787829563.239511:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787829563.239512:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829563.239515:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787829563.239517:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829563.239519:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829563.239523:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646842496:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829563.239526:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1787829563.239528:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1787829563.239530:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829563.239531:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1787829563.239535:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829563.239541:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787829563.239543:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417875968 : -126234291675648 : ffff8d30c9e2de00) 00000100:00000001:3.0:1787829563.239546:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1787829563.239559:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:3.0:1787829563.239564:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787829563.239573:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829563.239574:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829563.239577:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1787829563.239584:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829563.240566:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4299036182 00000100:00000001:3.0:1787829563.240575:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4299037440 00000100:00000001:3.0:1787829563.240583:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4299036183 00000100:00000001:3.0:1787829563.240586:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4299037440 00000100:00000001:3.0:1787829563.240588:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829563.240591:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829563.240594:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1787829563.240624:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829563/real 1787829563] req@ffff8d30c9e2de00 x1874670646842496/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829588 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1787829563.240649:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787829563.240651:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829563.240657:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829563.240659:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829563.240660:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829563.240662:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829563.240663:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829563.240667:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2de00 x1874670646842496/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829588 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829563.240671:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829563.240673:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829563.240676:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829563.240677:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829563.240679:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829563.240681:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829563.240683:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829563.240686:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829563.240690:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829563.240692:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829563.240696:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646842496:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829563.240701:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787829563.240703:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787829563.240705:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829563.240708:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787829563.240710:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829563.240713:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829563.240716:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2d2c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646842624:192.168.203.143@tcp:8:kworker.0 00000100:00000001:3.0:1787829563.240719:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1787829563.240722:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1787829563.240724:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829563.240726:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1787829563.240731:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829563.240740:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787829563.240741:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417873088 : -126234291678528 : ffff8d30c9e2d2c0) 00000100:00000001:3.0:1787829563.240746:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1787829563.240768:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:3.0:1787829563.240773:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787829563.240775:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829563.240776:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829563.240778:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1787829563.240780:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829563.240783:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4299036197 00000100:00000001:3.0:1787829563.240785:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4299037440 00000100:00000001:3.0:1787829563.240787:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4299036197 00000100:00000001:3.0:1787829563.240788:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4299037440 00000100:00000001:3.0:1787829563.240790:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829563.240791:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829563.240792:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1787829563.240795:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829563/real 1787829563] req@ffff8d30c9e2d2c0 x1874670646842624/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829588 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1787829563.240804:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787829563.240805:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829563.240807:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829563.240808:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829563.240809:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829563.240810:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829563.240811:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829563.240814:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2d2c0 x1874670646842624/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829588 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829563.240818:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829563.240819:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829563.240821:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829563.240822:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829563.240823:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829563.240824:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829563.240825:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829563.240826:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829563.240828:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829563.240830:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829563.240831:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2d2c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646842624:192.168.203.143@tcp:8:kworker.0 00000100:00000001:3.0:1787829563.240834:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829563.240835:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829563.240837:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829563.240838:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829563.240840:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829563.240841:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829563.240843:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829563.240845:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829563.240846:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829563.240849:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829563.240850:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829563.240852:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829563.240852:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829563.240853:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829563.240854:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829563.240855:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829563.240856:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829563.240857:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829563.240858:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829563.240860:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829563.240862:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829563.240863:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829563.240864:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829563.240865:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829563.240867:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829563.240868:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829563.240869:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829563.240870:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829563.240872:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829563.240873:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829563.240874:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829563.240876:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829563.240882:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787829563.240884:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829563.240886:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1787829563.240887:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829564.134713:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829564.134718:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829564.134777:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829564.134782:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829564.134786:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829565.158673:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829565.158679:0:815:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1787829565.158681:0:815:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1787829565.158683:0:815:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829565.158687:0:815:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1787829565.158689:0:815:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829565.158691:0:815:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829565.158692:0:815:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829565.158744:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829565.158747:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829565.158750:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829566.182713:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829566.182720:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829566.182780:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829566.182784:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829566.182786:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829567.206737:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829567.206749:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829567.206812:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829567.206817:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829567.206821:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829568.230733:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829568.230740:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829568.230820:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829568.230827:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829568.230832:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787829568.358688:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829568.358699:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1787829568.358703:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1787829568.358707:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787829568.358711:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1787829568.358718:0:40798:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 4368 00000100:00080000:1.0:1787829568.358729:0:40798:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1787829568.358732:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787829568.358735:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:1.0:1787829568.358739:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:1.0:1787829568.358742:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:1.0:1787829568.358745:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:1.0:1787829568.358747:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:1.0:1787829568.358750:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787829568.358751:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:1.0:1787829568.358756:0:40798:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:1.0:1787829568.358760:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829568.358762:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1787829568.358766:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829568.358769:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829568.358778:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:1.0:1787829568.358780:0:40798:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829568.358790:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1787829568.358792:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829568.358795:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1787829568.358797:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1787829568.358800:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829568.358810:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829568.358817:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d13efc00 x1874670646842752/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787829568.358825:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787829568.358828:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829568.358844:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1787829568.358847:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829568.358849:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1787829568.358850:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1787829568.358851:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787829568.358854:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1787829568.358856:0:40798:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4368 00000100:00080000:1.0:1787829568.358859:0:40798:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1787829568.358860:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787829568.358862:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:1.0:1787829568.358863:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:1.0:1787829568.358865:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:1.0:1787829568.358867:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000020:00000001:1.0:1787829568.358868:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:1.0:1787829568.358870:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787829568.358871:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:1.0:1787829568.358874:0:40798:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:1.0:1787829568.358876:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829568.358877:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:3.0:1787829568.358879:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:1.0:1787829568.358882:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829568.358884:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 02000000:00000001:1.0:1787829568.358886:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829568.358888:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787829568.358889:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:3.0:1787829568.358892:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000008:00100000:1.0:1787829568.358894:0:40798:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00000001:3.0:1787829568.358895:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:1.0:1787829568.358901:0:40798:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829568.358902:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:1.0:1787829568.358904:0:40798:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829568.358906:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829568.358908:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787829568.358909:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829568.358911:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829568.358914:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1787829568.358916:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00100000:3.0:1787829568.358917:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d13efc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646842752:192.168.203.143@tcp:250:kworker.0 02000000:00000001:1.0:1787829568.358918:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829568.358926:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829568.358926:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1787829568.358930:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1787829568.358932:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829568.358934:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00100000:1.0:1787829568.358939:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d13ed2c0 x1874670646842880/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:3.0:1787829568.358939:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829568.358948:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787829568.358950:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829568.358952:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1787829568.358955:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1787829568.358955:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787829568.358959:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839541353472 : -126234168198144 : ffff8d30d13efc00) 00000100:00080000:1.0:1787829568.358961:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:1.0:1787829568.358964:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:3.0:1787829568.358964:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00080000:1.0:1787829568.358968:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787829568.358973:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1787829568.358975:0:40798:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4368 00000100:00080000:1.0:1787829568.358979:0:40798:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1787829568.358980:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787829568.358982:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:1.0:1787829568.358983:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:1.0:1787829568.358984:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:1.0:1787829568.358986:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000020:00000001:1.0:1787829568.358988:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:1.0:1787829568.358989:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787829568.358990:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:1.0:1787829568.358993:0:40798:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:1.0:1787829568.358994:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829568.358996:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1787829568.358998:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000800:00000100:3.0:1787829568.359000:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 02000000:00000001:1.0:1787829568.359002:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829568.359004:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:1.0:1787829568.359006:0:40798:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1787829568.359008:0:40798:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829568.359010:0:40798:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829568.359013:0:2390:0:(events.c:34:request_out_callback()) Process entered 02000000:00000001:1.0:1787829568.359014:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1787829568.359016:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829568.359018:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1787829568.359019:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1787829568.359021:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829568.359021:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829568.359027:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829568.359028:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829568.359034:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00100000:1.0:1787829568.359037:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d13ed680 x1874670646843008/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787829568.359043:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829568.359044:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787829568.359047:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829568.359050:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829568.359052:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4299042560 00000100:00080000:1.0:1787829568.359053:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829568.359059:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1787829568.359059:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1787829568.359060:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787829568.359062:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1787829568.359064:0:40798:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4368 00000100:00000001:3.0:1787829568.359066:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 00000100:00080000:1.0:1787829568.359067:0:40798:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1787829568.359070:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787829568.359072:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:1.0:1787829568.359073:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:1.0:1787829568.359074:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:1.0:1787829568.359076:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000020:00000001:1.0:1787829568.359077:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:1.0:1787829568.359078:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787829568.359079:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:1.0:1787829568.359082:0:40798:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:1.0:1787829568.359083:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829568.359085:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:3.0:1787829568.359086:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4299042560 02000000:00000001:1.0:1787829568.359087:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829568.359092:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829568.359094:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:1.0:1787829568.359096:0:40798:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00000001:3.0:1787829568.359096:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 00000008:00000001:1.0:1787829568.359102:0:40798:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829568.359104:0:40798:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829568.359104:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829568.359108:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1787829568.359110:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829568.359110:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829568.359116:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1787829568.359117:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:3.0:1787829568.359117:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 02000000:00000001:1.0:1787829568.359121:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829568.359126:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829568.359129:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829568/real 1787829568] req@ffff8d30d13efc00 x1874670646842752/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829593 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1787829568.359133:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d13ef0c0 x1874670646843136/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000001:3.0:1787829568.359166:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000001:1.0:1787829568.359167:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000400:00000001:3.0:1787829568.359170:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787829568.359171:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829568.359175:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829568.359175:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829568.359179:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829568.359181:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829568.359183:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829568.359185:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829568.359190:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d13efc00 x1874670646842752/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829593 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829568.359197:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829568.359200:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829568.359202:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829568.359204:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829568.359205:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829568.359206:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829568.359207:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829568.359209:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829568.359213:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829568.359214:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829568.359216:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d13efc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646842752:192.168.203.143@tcp:250:kworker.0 00000100:00000001:3.0:1787829568.359220:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829568.359222:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829568.359223:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829568.359225:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829568.359227:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829568.359229:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829568.359231:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829568.359234:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829568.359235:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829568.359237:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829568.359238:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829568.359240:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829568.359246:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787829568.359247:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829568.359249:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1787829568.359250:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787829568.359251:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787829568.359252:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787829568.359253:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829568.359254:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787829568.359256:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829568.359258:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829568.359260:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d13ed2c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646842880:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829568.359262:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1787829568.359263:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1787829568.359265:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829568.359266:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1787829568.359268:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829568.359272:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787829568.359273:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839541342912 : -126234168208704 : ffff8d30d13ed2c0) 00000100:00000001:3.0:1787829568.359277:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1787829568.359295:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:3.0:1787829568.359299:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787829568.359300:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829568.359301:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829568.359302:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1787829568.359305:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829568.359308:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4299041302 00000100:00000001:3.0:1787829568.359310:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4299042560 00000100:00000001:3.0:1787829568.359311:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4299041302 00000100:00000001:3.0:1787829568.359313:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4299042560 00000100:00000001:3.0:1787829568.359314:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829568.359316:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829568.359317:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1787829568.359320:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829568/real 1787829568] req@ffff8d30d13ed2c0 x1874670646842880/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829593 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1787829568.359325:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787829568.359326:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829568.359327:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829568.359328:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829568.359329:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829568.359330:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829568.359331:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829568.359334:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d13ed2c0 x1874670646842880/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829593 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829568.359338:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829568.359340:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829568.359342:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829568.359343:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829568.359344:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829568.359345:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829568.359346:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829568.359348:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829568.359350:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829568.359351:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829568.359353:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d13ed2c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646842880:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829568.359355:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787829568.359356:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787829568.359357:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829568.359358:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787829568.359359:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829568.359361:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829568.359362:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d13ed680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646843008:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829568.359364:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1787829568.359365:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1787829568.359366:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829568.359367:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1787829568.359369:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829568.359372:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787829568.359373:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839541343872 : -126234168207744 : ffff8d30d13ed680) 00000100:00000001:3.0:1787829568.359375:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1787829568.359382:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:3.0:1787829568.359384:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787829568.359386:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829568.359387:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829568.359388:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1787829568.359389:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829568.359391:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4299041302 00000100:00000001:3.0:1787829568.359393:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4299042560 00000100:00000001:3.0:1787829568.359394:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4299041302 00000100:00000001:3.0:1787829568.359396:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4299042560 00000100:00000001:3.0:1787829568.359397:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829568.359399:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829568.359400:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1787829568.359402:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829568/real 1787829568] req@ffff8d30d13ed680 x1874670646843008/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829593 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1787829568.359407:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787829568.359407:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829568.359409:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829568.359410:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829568.359410:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829568.359411:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829568.359412:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829568.359414:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d13ed680 x1874670646843008/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829593 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829568.359419:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829568.359420:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829568.359422:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829568.359423:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829568.359424:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829568.359425:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829568.359426:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829568.359427:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829568.359429:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829568.359430:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829568.359432:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d13ed680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646843008:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829568.359434:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787829568.359435:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787829568.359436:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829568.359438:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787829568.359439:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829568.359440:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829568.359442:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d13ef0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646843136:192.168.203.143@tcp:8:kworker.0 00000100:00000001:3.0:1787829568.359443:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1787829568.359444:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1787829568.359445:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829568.359446:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1787829568.359448:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829568.359451:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787829568.359452:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839541350592 : -126234168201024 : ffff8d30d13ef0c0) 00000100:00000001:3.0:1787829568.359454:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1787829568.359460:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:3.0:1787829568.359462:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787829568.359463:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829568.359464:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829568.359465:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1787829568.359467:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829568.359469:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4299041315 00000100:00000001:3.0:1787829568.359470:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4299042560 00000100:00000001:3.0:1787829568.359472:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4299041315 00000100:00000001:3.0:1787829568.359473:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4299042560 00000100:00000001:3.0:1787829568.359474:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829568.359476:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829568.359477:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1787829568.359479:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829568/real 1787829568] req@ffff8d30d13ef0c0 x1874670646843136/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829593 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1787829568.359483:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787829568.359484:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829568.359485:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829568.359486:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829568.359487:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829568.359488:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829568.359489:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829568.359491:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d13ef0c0 x1874670646843136/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829593 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829568.359502:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829568.359503:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829568.359505:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829568.359506:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829568.359507:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829568.359508:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829568.359510:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829568.359511:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829568.359513:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829568.359515:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829568.359517:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d13ef0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646843136:192.168.203.143@tcp:8:kworker.0 00000100:00000001:3.0:1787829568.359519:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829568.359520:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829568.359521:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829568.359521:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829568.359523:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829568.359524:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829568.359524:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829568.359527:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829568.359528:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829568.359529:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829568.359530:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829568.359531:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829568.359532:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829568.359533:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829568.359534:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829568.359535:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829568.359536:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829568.359536:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829568.359537:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829568.359538:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829568.359539:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829568.359540:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829568.359541:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829568.359542:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829568.359543:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829568.359544:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829568.359545:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829568.359546:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829568.359546:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829568.359547:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829568.359548:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829568.359549:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829568.359551:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787829568.359552:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829568.359554:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1787829568.359555:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829569.254703:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829569.254709:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829569.254759:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829569.254764:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829569.254774:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829570.278697:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829570.278702:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829570.278743:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829570.278747:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829570.278751:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829571.302724:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829571.302732:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829571.302795:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829571.302799:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829571.302803:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829572.326756:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829572.326764:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829572.326836:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829572.326842:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829572.326846:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829573.350693:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829573.350700:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829573.350755:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829573.350758:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829573.350762:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787829573.478690:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829573.478701:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1787829573.478703:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1787829573.478707:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787829573.478711:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1787829573.478719:0:40798:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 4373 00000100:00080000:1.0:1787829573.478730:0:40798:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1787829573.478734:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787829573.478736:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:1.0:1787829573.478751:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:1.0:1787829573.478754:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:1.0:1787829573.478756:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:1.0:1787829573.478758:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:1.0:1787829573.478760:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787829573.478761:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:1.0:1787829573.478765:0:40798:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:1.0:1787829573.478767:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829573.478769:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1787829573.478772:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829573.478775:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829573.478776:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:1.0:1787829573.478779:0:40798:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829573.478788:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1787829573.478789:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829573.478792:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1787829573.478794:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1787829573.478798:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829573.478807:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829573.478814:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d13ecb40 x1874670646843264/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787829573.478822:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787829573.478826:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829573.478839:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1787829573.478841:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829573.478844:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1787829573.478845:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1787829573.478846:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787829573.478848:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1787829573.478850:0:40798:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4373 00000100:00080000:1.0:1787829573.478852:0:40798:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1787829573.478854:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787829573.478855:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:1.0:1787829573.478857:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:1.0:1787829573.478858:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:1.0:1787829573.478860:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000020:00000001:1.0:1787829573.478862:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:1.0:1787829573.478864:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787829573.478865:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:1.0:1787829573.478868:0:40798:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:1.0:1787829573.478870:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829573.478870:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:1.0:1787829573.478874:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:3.0:1787829573.478875:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 02000000:00000001:1.0:1787829573.478879:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829573.478879:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 02000000:00000001:1.0:1787829573.478884:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829573.478884:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787829573.478887:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829573.478888:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:1.0:1787829573.478893:0:40798:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 02000000:00000001:3.0:1787829573.478894:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000008:00000001:1.0:1787829573.478898:0:40798:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829573.478899:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829573.478900:0:40798:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829573.478904:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829573.478906:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1787829573.478908:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829573.478911:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00100000:3.0:1787829573.478912:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d13ecb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646843264:192.168.203.143@tcp:250:kworker.0 02000000:00000001:1.0:1787829573.478914:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1787829573.478918:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829573.478919:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1787829573.478923:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1787829573.478925:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829573.478927:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829573.478928:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1787829573.478934:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829573.478938:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d13ec000 x1874670646843392/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787829573.478949:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787829573.478953:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839541340992 : -126234168210624 : ffff8d30d13ecb40) 00000100:00000001:3.0:1787829573.478958:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:1.0:1787829573.478960:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787829573.478964:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829573.478969:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1787829573.478972:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829573.478975:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:1.0:1787829573.478977:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1787829573.478978:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787829573.478981:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1787829573.478983:0:40798:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4373 00000100:00080000:1.0:1787829573.478987:0:40798:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1787829573.478990:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000800:00000100:3.0:1787829573.478991:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787829573.478992:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:1.0:1787829573.478996:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:1.0:1787829573.478998:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:1.0:1787829573.478999:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000020:00000001:1.0:1787829573.479001:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:1.0:1787829573.479003:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787829573.479004:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:3.0:1787829573.479005:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00080000:1.0:1787829573.479009:0:40798:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:3.0:1787829573.479011:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829573.479012:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829573.479016:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:3.0:1787829573.479016:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829573.479020:0:2390:0:(events.c:68:request_out_callback()) Process leaving 02000000:00000001:1.0:1787829573.479021:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829573.479024:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829573.479026:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:3.0:1787829573.479026:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000008:00100000:1.0:1787829573.479031:0:40798:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00000001:3.0:1787829573.479031:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4299047680 00000008:00000001:1.0:1787829573.479035:0:40798:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829573.479037:0:40798:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829573.479042:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:3.0:1787829573.479042:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 02000000:00000001:1.0:1787829573.479043:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829573.479048:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1787829573.479049:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1787829573.479050:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829573.479055:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829573.479058:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d13ec780 x1874670646843520/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787829573.479059:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4299047680 00000100:00000001:3.0:1787829573.479067:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 00000100:00000001:1.0:1787829573.479068:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787829573.479070:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829573.479071:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829573.479077:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829573.479078:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829573.479081:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00080000:1.0:1787829573.479082:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829573.479086:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1787829573.479088:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00100000:3.0:1787829573.479090:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829573/real 1787829573] req@ffff8d30d13ecb40 x1874670646843264/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829598 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1787829573.479091:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787829573.479095:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1787829573.479102:0:40798:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4373 00000400:00000001:3.0:1787829573.479105:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787829573.479107:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00080000:1.0:1787829573.479108:0:40798:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1787829573.479123:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829573.479125:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787829573.479126:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:3.0:1787829573.479129:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000020:00000001:1.0:1787829573.479130:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:3.0:1787829573.479131:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:1.0:1787829573.479134:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:3.0:1787829573.479135:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829573.479138:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:1.0:1787829573.479139:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00100000:3.0:1787829573.479143:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d13ecb40 x1874670646843264/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829598 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:1.0:1787829573.479145:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:1.0:1787829573.479152:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787829573.479156:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:3.0:1787829573.479157:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829573.479161:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787829573.479166:0:40798:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:3.0:1787829573.479166:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829573.479170:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829573.479172:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829573.479173:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:1.0:1787829573.479176:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:3.0:1787829573.479178:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 02000000:00000001:1.0:1787829573.479181:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829573.479181:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 02000000:00000001:1.0:1787829573.479187:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787829573.479188:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00000001:1.0:1787829573.479190:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:1.0:1787829573.479192:0:40798:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000008:00000001:1.0:1787829573.479195:0:40798:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829573.479198:0:40798:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787829573.479201:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829573.479203:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 02000000:00000001:1.0:1787829573.479205:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00100000:3.0:1787829573.479208:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d13ecb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646843264:192.168.203.143@tcp:250:kworker.0 02000000:00000001:1.0:1787829573.479209:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829573.479214:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:3.0:1787829573.479214:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829573.479217:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 02000000:00000001:1.0:1787829573.479218:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:3.0:1787829573.479219:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829573.479223:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787829573.479224:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829573.479225:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829573.479229:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829573.479231:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 00000100:00000001:1.0:1787829573.479233:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829573.479235:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829573.479239:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829573.479241:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829573.479244:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787829573.479245:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d13ec3c0 x1874670646843648/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787829573.479247:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829573.479253:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787829573.479254:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829573.479256:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1787829573.479258:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787829573.479260:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787829573.479262:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787829573.479264:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829573.479268:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787829573.479271:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829573.479275:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829573.479276:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00100000:3.0:1787829573.479279:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d13ec000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646843392:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787829573.479283:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829573.479284:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1787829573.479288:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:1.0:1787829573.479290:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829573.479290:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829573.479297:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1787829573.479300:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829573.479305:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787829573.479306:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839541338112 : -126234168213504 : ffff8d30d13ec000) 00000100:00000001:3.0:1787829573.479310:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1787829573.479321:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:3.0:1787829573.479324:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787829573.479327:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829573.479328:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829573.479329:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1787829573.479331:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829573.479334:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4299046422 00000100:00000001:3.0:1787829573.479336:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4299047680 00000100:00000001:3.0:1787829573.479338:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4299046422 00000100:00000001:3.0:1787829573.479339:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4299047680 00000100:00000001:3.0:1787829573.479341:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829573.479342:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829573.479343:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1787829573.479346:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829573/real 1787829573] req@ffff8d30d13ec000 x1874670646843392/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829598 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1787829573.479352:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787829573.479352:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829573.479354:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829573.479355:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829573.479356:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829573.479357:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829573.479358:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829573.479361:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d13ec000 x1874670646843392/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829598 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829573.479366:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829573.479367:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829573.479369:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829573.479370:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829573.479370:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829573.479371:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829573.479373:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829573.479374:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829573.479376:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829573.479377:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829573.479380:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d13ec000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646843392:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829573.479382:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787829573.479383:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787829573.479384:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829573.479386:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787829573.479387:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829573.479389:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829573.479391:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d13ec780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646843520:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829573.479395:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1787829573.479397:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1787829573.479398:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829573.479400:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1787829573.479403:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829573.479408:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787829573.479410:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839541340032 : -126234168211584 : ffff8d30d13ec780) 00000100:00000001:3.0:1787829573.479414:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1787829573.479436:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:3.0:1787829573.479441:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787829573.479444:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829573.479445:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829573.479447:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1787829573.479450:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829573.479453:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4299046422 00000100:00000001:3.0:1787829573.479456:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4299047680 00000100:00000001:3.0:1787829573.479459:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4299046422 00000100:00000001:3.0:1787829573.479460:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4299047680 00000100:00000001:3.0:1787829573.479462:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829573.479463:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829573.479465:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1787829573.479468:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829573/real 1787829573] req@ffff8d30d13ec780 x1874670646843520/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829598 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1787829573.479479:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787829573.479480:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829573.479482:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829573.479483:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829573.479484:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829573.479485:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829573.479486:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829573.479488:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d13ec780 x1874670646843520/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829598 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829573.479493:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829573.479494:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829573.479496:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829573.479497:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829573.479497:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829573.479498:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829573.479500:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829573.479501:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829573.479510:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829573.479511:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829573.479513:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d13ec780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646843520:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829573.479516:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829573.479517:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829573.479518:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829573.479519:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829573.479521:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829573.479522:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829573.479523:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829573.479525:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829573.479525:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829573.479527:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829573.479528:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829573.479529:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829573.479530:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829573.479531:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829573.479532:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829573.479533:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829573.479533:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829573.479534:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829573.479535:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829573.479536:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829573.479537:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829573.479538:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829573.479540:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787829573.479541:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829573.479542:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1787829573.479543:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787829573.479544:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787829573.479545:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787829573.479545:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829573.479547:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787829573.479548:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829573.479549:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829573.479551:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d13ec3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646843648:192.168.203.143@tcp:8:kworker.0 00000100:00000001:3.0:1787829573.479554:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1787829573.479555:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1787829573.479556:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829573.479557:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1787829573.479559:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829573.479563:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787829573.479564:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839541339072 : -126234168212544 : ffff8d30d13ec3c0) 00000100:00000001:3.0:1787829573.479567:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1787829573.479575:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:3.0:1787829573.479579:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787829573.479580:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829573.479581:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829573.479582:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1787829573.479584:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829573.479586:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4299046435 00000100:00000001:3.0:1787829573.479588:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4299047680 00000100:00000001:3.0:1787829573.479595:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4299046435 00000100:00000001:3.0:1787829573.479597:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4299047680 00000100:00000001:3.0:1787829573.479599:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829573.479627:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829573.479628:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1787829573.479631:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829573/real 1787829573] req@ffff8d30d13ec3c0 x1874670646843648/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829598 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1787829573.479637:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787829573.479638:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829573.479640:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829573.479641:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829573.479642:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829573.479643:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829573.479644:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829573.479647:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d13ec3c0 x1874670646843648/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829598 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829573.479651:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829573.479652:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829573.479653:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829573.479654:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829573.479655:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829573.479656:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829573.479658:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829573.479660:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829573.479662:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829573.479663:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829573.479665:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d13ec3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646843648:192.168.203.143@tcp:8:kworker.0 00000100:00000001:3.0:1787829573.479667:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829573.479668:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829573.479669:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829573.479670:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829573.479672:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829573.479673:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829573.479674:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829573.479675:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829573.479676:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829573.479677:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829573.479678:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829573.479679:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829573.479681:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787829573.479682:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829573.479683:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1787829573.479685:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829574.374710:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829574.374716:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829574.374782:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829574.374786:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829574.374790:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829575.398689:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829575.398704:0:815:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1787829575.398707:0:815:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1787829575.398710:0:815:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829575.398715:0:815:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1787829575.398718:0:815:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829575.398719:0:815:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829575.398721:0:815:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829575.398786:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829575.398790:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829575.398793:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829576.422692:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829576.422698:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829576.422760:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829576.422764:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829576.422768:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829577.446682:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829577.446687:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829577.446734:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829577.446737:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829577.446741:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829578.470705:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829578.470710:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829578.470761:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829578.470764:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829578.470767:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787829578.598670:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829578.598679:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1787829578.598681:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787829578.598684:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787829578.598688:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787829578.598694:0:40798:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 4379 00000100:00080000:2.0:1787829578.598702:0:40798:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787829578.598705:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829578.598706:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787829578.598709:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787829578.598712:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787829578.598714:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:2.0:1787829578.598716:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:2.0:1787829578.598717:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829578.598719:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:2.0:1787829578.598722:0:40798:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787829578.598724:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829578.598726:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787829578.598728:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787829578.598730:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829578.598732:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:2.0:1787829578.598734:0:40798:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829578.598743:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787829578.598744:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829578.598746:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787829578.598748:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787829578.598752:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829578.598761:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787829578.598768:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2c3c0 x1874670646843776/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787829578.598776:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787829578.598779:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829578.598790:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787829578.598793:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829578.598795:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1787829578.598796:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787829578.598797:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787829578.598799:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787829578.598801:0:40798:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4379 00000100:00080000:2.0:1787829578.598803:0:40798:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787829578.598805:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829578.598806:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787829578.598808:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787829578.598809:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787829578.598811:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000020:00000001:2.0:1787829578.598813:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:2.0:1787829578.598815:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829578.598816:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:2.0:1787829578.598818:0:40798:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787829578.598819:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829578.598820:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:3.0:1787829578.598823:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:2.0:1787829578.598825:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787829578.598829:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829578.598830:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:3.0:1787829578.598831:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000008:00100000:2.0:1787829578.598834:0:40798:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00000001:3.0:1787829578.598835:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000008:00000001:2.0:1787829578.598837:0:40798:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829578.598840:0:40798:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829578.598840:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787829578.598845:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829578.598846:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787829578.598848:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829578.598851:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787829578.598852:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787829578.598853:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829578.598853:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787829578.598858:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787829578.598859:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829578.598865:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787829578.598867:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2cb40 x1874670646843904/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787829578.598874:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646843776:192.168.203.143@tcp:250:kworker.0 00000100:00000001:2.0:1787829578.598876:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787829578.598880:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829578.598883:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:2.0:1787829578.598884:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787829578.598887:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 02000000:00000001:3.0:1787829578.598888:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000100:00080000:2.0:1787829578.598891:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 02000000:00000001:3.0:1787829578.598893:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829578.598894:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787829578.598896:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 02000000:00000001:3.0:1787829578.598899:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:2.0:1787829578.598900:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787829578.598905:0:40798:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4379 00000100:00080000:2.0:1787829578.598908:0:40798:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 02000000:00000001:3.0:1787829578.598908:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829578.598910:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829578.598913:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787829578.598914:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787829578.598915:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787829578.598916:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000020:00000001:2.0:1787829578.598918:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:2.0:1787829578.598919:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829578.598920:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:2.0:1787829578.598923:0:40798:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787829578.598925:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829578.598925:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 02000000:00000001:2.0:1787829578.598929:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:3.0:1787829578.598930:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417869248 : -126234291682368 : ffff8d30c9e2c3c0) 02000000:00000001:2.0:1787829578.598931:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787829578.598936:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829578.598938:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:3.0:1787829578.598938:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000008:00100000:2.0:1787829578.598942:0:40798:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1787829578.598944:0:40798:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829578.598945:0:40798:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829578.598949:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787829578.598950:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829578.598951:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787829578.598952:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787829578.598953:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829578.598956:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787829578.598959:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2fc00 x1874670646844032/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787829578.598965:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787829578.598966:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829578.598969:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787829578.598970:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829578.598972:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1787829578.598973:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787829578.598975:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787829578.598976:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787829578.598981:0:40798:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4379 00000800:00000100:3.0:1787829578.598984:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00080000:2.0:1787829578.598985:0:40798:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787829578.598989:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829578.598990:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787829578.598991:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787829578.598993:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787829578.598994:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000020:00000001:2.0:1787829578.598995:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:2.0:1787829578.598997:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829578.598998:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:2.0:1787829578.599000:0:40798:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:3.0:1787829578.599001:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1787829578.599002:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829578.599005:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787829578.599008:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829578.599008:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 02000000:00000001:2.0:1787829578.599010:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829578.599013:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:3.0:1787829578.599013:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000008:00100000:2.0:1787829578.599015:0:40798:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00000001:3.0:1787829578.599018:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000008:00000001:2.0:1787829578.599019:0:40798:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829578.599022:0:40798:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829578.599025:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:3.0:1787829578.599026:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829578.599027:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829578.599032:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787829578.599033:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:3.0:1787829578.599034:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4299052800 02000000:00000001:2.0:1787829578.599035:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829578.599040:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787829578.599043:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2c780 x1874670646844160/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787829578.599048:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787829578.599049:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829578.599050:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829578.599058:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 00000100:00000001:3.0:1787829578.599070:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4299052800 00000100:00000001:3.0:1787829578.599087:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 00000100:00000001:3.0:1787829578.599093:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829578.599096:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829578.599099:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1787829578.599107:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829578/real 1787829578] req@ffff8d30c9e2c3c0 x1874670646843776/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829603 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1787829578.599119:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787829578.599121:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829578.599125:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829578.599135:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829578.599138:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829578.599141:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829578.599144:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829578.599149:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2c3c0 x1874670646843776/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829603 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829578.599158:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829578.599161:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829578.599165:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829578.599167:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829578.599169:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829578.599171:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829578.599174:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829578.599177:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829578.599182:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829578.599185:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829578.599189:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646843776:192.168.203.143@tcp:250:kworker.0 00000100:00000001:3.0:1787829578.599195:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829578.599197:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829578.599200:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829578.599202:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829578.599207:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829578.599209:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829578.599211:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829578.599217:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829578.599219:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829578.599223:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829578.599225:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829578.599228:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829578.599235:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787829578.599238:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829578.599241:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1787829578.599242:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787829578.599244:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787829578.599246:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787829578.599247:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829578.599250:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787829578.599252:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829578.599254:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829578.599259:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646843904:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829578.599262:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1787829578.599264:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1787829578.599266:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829578.599269:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1787829578.599272:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829578.599279:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787829578.599281:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417871168 : -126234291680448 : ffff8d30c9e2cb40) 00000100:00000001:3.0:1787829578.599286:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1787829578.599300:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:3.0:1787829578.599306:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787829578.599310:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829578.599312:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829578.599314:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1787829578.599318:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829578.599393:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4299051542 00000100:00000001:3.0:1787829578.599398:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4299052800 00000100:00000001:3.0:1787829578.599400:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4299051542 00000100:00000001:3.0:1787829578.599403:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4299052800 00000100:00000001:3.0:1787829578.599510:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829578.599514:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829578.599516:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1787829578.599522:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829578/real 1787829578] req@ffff8d30c9e2cb40 x1874670646843904/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829603 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1787829578.599533:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787829578.599535:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829578.599575:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829578.599579:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829578.599581:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829578.599584:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829578.599586:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829578.599593:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2cb40 x1874670646843904/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829603 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829578.599649:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829578.599652:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829578.599655:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829578.599657:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829578.599659:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829578.599661:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829578.599663:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829578.599665:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829578.599669:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829578.599672:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829578.599676:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646843904:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829578.599680:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787829578.599691:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787829578.599693:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829578.599696:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787829578.599698:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829578.599700:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829578.599704:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646844032:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829578.599708:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1787829578.599710:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1787829578.599712:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829578.599713:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1787829578.599717:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829578.599723:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787829578.599725:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417883648 : -126234291667968 : ffff8d30c9e2fc00) 00000100:00000001:3.0:1787829578.599729:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1787829578.599743:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:3.0:1787829578.599749:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787829578.599751:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829578.599753:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829578.599755:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1787829578.599757:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829578.599761:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4299051543 00000100:00000001:3.0:1787829578.599764:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4299052800 00000100:00000001:3.0:1787829578.599766:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4299051543 00000100:00000001:3.0:1787829578.599768:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4299052800 00000100:00000001:3.0:1787829578.599770:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829578.599772:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829578.599774:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1787829578.599778:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829578/real 1787829578] req@ffff8d30c9e2fc00 x1874670646844032/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829603 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1787829578.599793:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787829578.599794:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829578.599796:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829578.599798:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829578.599799:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829578.599801:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829578.599803:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829578.599806:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2fc00 x1874670646844032/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829603 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829578.599813:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829578.599815:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829578.599817:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829578.599819:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829578.599820:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829578.599822:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829578.599824:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829578.599826:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829578.599829:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829578.599831:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829578.599834:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646844032:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829578.599838:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787829578.599840:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787829578.599841:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829578.599843:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787829578.599845:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829578.599847:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829578.599850:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646844160:192.168.203.143@tcp:8:kworker.0 00000100:00000001:3.0:1787829578.599853:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1787829578.599854:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1787829578.599856:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829578.599858:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1787829578.599860:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829578.599865:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787829578.599866:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417870208 : -126234291681408 : ffff8d30c9e2c780) 00000100:00000001:3.0:1787829578.599870:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1787829578.599879:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:3.0:1787829578.599884:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787829578.599886:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829578.599887:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829578.599889:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1787829578.599891:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829578.599894:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4299051556 00000100:00000001:3.0:1787829578.599896:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4299052800 00000100:00000001:3.0:1787829578.599898:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4299051556 00000100:00000001:3.0:1787829578.599901:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4299052800 00000100:00000001:3.0:1787829578.599902:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829578.599904:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829578.599906:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1787829578.599910:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829578/real 1787829578] req@ffff8d30c9e2c780 x1874670646844160/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829603 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1787829578.599917:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787829578.599919:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829578.599921:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829578.599922:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829578.599923:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829578.599925:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829578.599927:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829578.599931:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2c780 x1874670646844160/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829603 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829578.599938:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829578.599940:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829578.599943:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829578.599945:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829578.599946:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829578.599948:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829578.599950:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829578.599953:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829578.599956:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829578.599959:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829578.599962:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646844160:192.168.203.143@tcp:8:kworker.0 00000100:00000001:3.0:1787829578.599966:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829578.599968:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829578.599969:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829578.599971:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829578.599973:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829578.599974:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829578.599976:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829578.599978:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829578.599980:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829578.599982:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829578.599983:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829578.599985:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829578.599986:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829578.599987:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829578.599989:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829578.599990:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829578.599991:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829578.599993:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829578.599994:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829578.599997:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829578.599998:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829578.600000:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829578.600001:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829578.600002:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829578.600004:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829578.600005:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829578.600007:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829578.600008:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829578.600010:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829578.600012:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829578.600013:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829578.600015:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829578.600019:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787829578.600020:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829578.600022:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1787829578.600023:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829579.494709:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829579.494714:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829579.494816:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829579.494820:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829579.494825:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829580.518681:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829580.518687:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829580.518737:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829580.518742:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829580.518746:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829581.542701:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829581.542706:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829581.542792:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829581.542800:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829581.542804:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829582.566700:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829582.566705:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829582.566763:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829582.566768:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829582.566772:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829583.590903:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829583.590915:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829583.591140:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829583.591152:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829583.591156:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787829583.718763:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829583.718773:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1787829583.718776:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787829583.718779:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787829583.718782:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787829583.718788:0:40798:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 4384 00000100:00080000:2.0:1787829583.718796:0:40798:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787829583.718798:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829583.718800:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787829583.718803:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787829583.718806:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787829583.718808:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:2.0:1787829583.718810:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:2.0:1787829583.718812:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829583.718813:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:2.0:1787829583.718816:0:40798:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787829583.718818:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829583.718820:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787829583.718824:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787829583.718828:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829583.718830:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:2.0:1787829583.718833:0:40798:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829583.718845:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787829583.718847:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829583.718852:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787829583.718855:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787829583.718859:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829583.718871:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787829583.718880:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2f0c0 x1874670646844288/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787829583.718900:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787829583.718905:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829583.718918:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787829583.718921:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829583.718923:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1787829583.718924:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787829583.718926:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787829583.718928:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787829583.718930:0:40798:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4384 00000100:00080000:2.0:1787829583.718933:0:40798:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787829583.718935:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829583.718936:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787829583.718939:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787829583.718940:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787829583.718941:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000020:00000001:2.0:1787829583.718943:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:2.0:1787829583.718945:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829583.718946:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:2.0:1787829583.718949:0:40798:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787829583.718951:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829583.718953:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:3.0:1787829583.718957:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:2.0:1787829583.718958:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787829583.718962:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829583.718964:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1787829583.718968:0:40798:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1787829583.718970:0:40798:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829583.718972:0:40798:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829583.718975:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787829583.718976:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829583.718977:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787829583.718978:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787829583.718980:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829583.718980:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1787829583.718985:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829583.718985:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787829583.718990:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00100000:2.0:1787829583.718993:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2ed00 x1874670646844416/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787829583.718994:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829583.719001:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:2.0:1787829583.719004:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 02000000:00000001:3.0:1787829583.719006:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787829583.719007:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829583.719013:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829583.719014:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787829583.719019:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829583.719024:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:3.0:1787829583.719024:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646844288:192.168.203.143@tcp:250:kworker.0 00000100:00000001:2.0:1787829583.719027:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787829583.719029:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787829583.719034:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:3.0:1787829583.719035:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00080000:2.0:1787829583.719051:0:40798:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4384 02000000:00000001:3.0:1787829583.719052:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1787829583.719056:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787829583.719057:0:40798:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 02000000:00000001:3.0:1787829583.719061:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:2.0:1787829583.719062:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829583.719066:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787829583.719067:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787829583.719068:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787829583.719069:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000020:00000001:2.0:1787829583.719071:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:2.0:1787829583.719072:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 02000000:00000001:3.0:1787829583.719082:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829583.719099:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787829583.719101:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417880768 : -126234291670848 : ffff8d30c9e2f0c0) 00000100:00000001:3.0:1787829583.719105:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1787829583.719138:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:3.0:1787829583.719159:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787829583.719163:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829583.719165:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829583.719166:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1787829583.719170:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829583.719173:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4299057920 00000100:00000001:3.0:1787829583.719187:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 00000100:00000001:3.0:1787829583.719198:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4299057920 00000100:00000001:3.0:1787829583.719201:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 00000100:00000001:3.0:1787829583.719825:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829583.719828:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829583.719830:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1787829583.719836:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829583/real 1787829583] req@ffff8d30c9e2f0c0 x1874670646844288/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829608 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1787829583.719845:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787829583.719847:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829583.719850:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829583.719852:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829583.719853:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829583.719855:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829583.719856:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829583.719859:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2f0c0 x1874670646844288/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829608 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829583.719865:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829583.719867:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829583.719870:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829583.719870:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829583.719871:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829583.719873:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829583.719874:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829583.719876:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829583.719879:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829583.719880:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829583.719883:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646844288:192.168.203.143@tcp:250:kworker.0 00000100:00000001:3.0:1787829583.719886:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829583.719888:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829583.719890:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829583.719892:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829583.719895:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829583.719897:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829583.719898:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829583.719901:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829583.719903:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829583.719907:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829583.719907:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829583.719909:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829583.719916:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787829583.719918:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829583.719919:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1787829583.719920:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787829583.719921:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787829583.719922:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787829583.719923:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829583.719925:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787829583.719927:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829583.719929:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829583.719931:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646844416:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829583.719933:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1787829583.719934:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1787829583.719936:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829583.719937:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1787829583.719939:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829583.719943:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787829583.719944:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417879808 : -126234291671808 : ffff8d30c9e2ed00) 00000100:00000001:3.0:1787829583.719948:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1787829583.719959:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:3.0:1787829583.719963:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787829583.719964:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829583.719965:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829583.719966:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1787829583.719968:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829583.719971:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4299056662 00000100:00000001:3.0:1787829583.719973:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4299057920 00000100:00000001:3.0:1787829583.719975:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4299056662 00000100:00000001:3.0:1787829583.719977:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4299057920 00000100:00000001:3.0:1787829583.719978:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829583.719979:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829583.719981:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1787829583.719984:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829583/real 1787829583] req@ffff8d30c9e2ed00 x1874670646844416/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829608 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1787829583.719989:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787829583.719990:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829583.719991:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829583.719992:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829583.719993:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829583.719994:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829583.719995:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829583.719998:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2ed00 x1874670646844416/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829608 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829583.720002:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829583.720003:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829583.720005:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829583.720006:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829583.720006:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829583.720008:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829583.720009:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829583.720010:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829583.720012:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829583.720013:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829583.720015:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646844416:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829583.720017:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829583.720018:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829583.720019:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829583.720020:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829583.720021:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829583.720023:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829583.720024:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829583.720025:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829583.720026:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829583.720029:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829583.720030:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829583.720031:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829583.720033:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787829583.720034:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829583.720035:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1787829583.720036:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829583.720217:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:2.0:1787829583.720224:0:40798:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787829583.720226:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829583.720228:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787829583.720232:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787829583.720234:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829583.720237:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1787829583.720239:0:40798:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1787829583.720242:0:40798:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829583.720244:0:40798:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829583.720249:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787829583.720250:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829583.720251:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787829583.720252:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787829583.720253:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829583.720257:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787829583.720261:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2e940 x1874670646844544/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787829583.720266:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787829583.720268:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829583.720277:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787829583.720279:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829583.720281:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1787829583.720282:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787829583.720283:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787829583.720285:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787829583.720287:0:40798:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4384 00000100:00080000:2.0:1787829583.720291:0:40798:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787829583.720293:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829583.720294:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787829583.720295:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787829583.720297:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:3.0:1787829583.720298:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:2.0:1787829583.720299:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:3.0:1787829583.720301:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000020:00000001:2.0:1787829583.720303:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:3.0:1787829583.720305:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1787829583.720307:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829583.720308:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787829583.720311:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:3.0:1787829583.720311:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829583.720317:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000100:00080000:2.0:1787829583.720318:0:40798:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 02000000:00000001:3.0:1787829583.720320:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787829583.720321:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829583.720323:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829583.720324:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787829583.720329:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00100000:3.0:1787829583.720329:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646844544:192.168.203.143@tcp:38:kworker.0 02000000:00000001:2.0:1787829583.720332:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829583.720335:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:2.0:1787829583.720336:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 02000000:00000001:3.0:1787829583.720339:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000008:00100000:2.0:1787829583.720340:0:40798:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 02000000:00000001:3.0:1787829583.720342:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:2.0:1787829583.720344:0:40798:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829583.720346:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:2.0:1787829583.720347:0:40798:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829583.720350:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829583.720352:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787829583.720354:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829583.720356:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787829583.720357:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787829583.720359:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829583.720359:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787829583.720362:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417878848 : -126234291672768 : ffff8d30c9e2e940) 00000100:00000001:3.0:1787829583.720366:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:2.0:1787829583.720367:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787829583.720373:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2c000 x1874670646844672/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787829583.720381:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000800:00000100:3.0:1787829583.720381:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:2.0:1787829583.720384:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829583.720388:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1787829583.720389:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829583.720393:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829583.720396:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829583.720398:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1787829583.720401:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829583.720403:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4299056663 00000100:00000001:3.0:1787829583.720405:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4299057920 00000100:00000001:3.0:1787829583.720409:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4299056663 00000100:00000001:3.0:1787829583.720411:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4299057920 00000100:00000001:3.0:1787829583.720412:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829583.720414:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829583.720415:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1787829583.720418:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829583/real 1787829583] req@ffff8d30c9e2e940 x1874670646844544/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829608 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1787829583.720423:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787829583.720424:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829583.720426:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829583.720427:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829583.720428:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829583.720429:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829583.720430:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829583.720432:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2e940 x1874670646844544/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829608 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829583.720436:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829583.720438:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829583.720440:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829583.720441:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829583.720442:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829583.720443:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829583.720444:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829583.720446:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829583.720448:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829583.720449:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829583.720451:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646844544:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829583.720453:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829583.720454:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829583.720455:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829583.720456:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829583.720458:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829583.720459:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829583.720460:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829583.720461:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829583.720462:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829583.720463:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829583.720464:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829583.720465:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829583.720468:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787829583.720469:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829583.720470:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1787829583.720471:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787829583.720472:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787829583.720473:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787829583.720474:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829583.720475:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787829583.720476:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829583.720477:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829583.720479:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646844672:192.168.203.143@tcp:8:kworker.0 00000100:00000001:3.0:1787829583.720481:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1787829583.720482:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1787829583.720483:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829583.720485:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1787829583.720486:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829583.720490:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787829583.720491:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417868288 : -126234291683328 : ffff8d30c9e2c000) 00000100:00000001:3.0:1787829583.720493:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1787829583.720500:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:3.0:1787829583.720502:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787829583.720503:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829583.720504:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829583.720505:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1787829583.720507:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829583.720509:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4299056676 00000100:00000001:3.0:1787829583.720511:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4299057920 00000100:00000001:3.0:1787829583.720512:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4299056676 00000100:00000001:3.0:1787829583.720514:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4299057920 00000100:00000001:3.0:1787829583.720515:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829583.720516:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829583.720518:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1787829583.720521:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829583/real 1787829583] req@ffff8d30c9e2c000 x1874670646844672/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829608 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1787829583.720525:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787829583.720526:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829583.720527:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829583.720528:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829583.720529:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829583.720530:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829583.720531:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829583.720533:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2c000 x1874670646844672/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829608 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829583.720537:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829583.720539:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829583.720540:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829583.720541:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829583.720542:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829583.720543:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829583.720544:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829583.720545:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829583.720547:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829583.720548:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829583.720550:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646844672:192.168.203.143@tcp:8:kworker.0 00000100:00000001:3.0:1787829583.720552:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829583.720553:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829583.720554:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829583.720555:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829583.720556:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829583.720557:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829583.720558:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829583.720560:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829583.720560:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829583.720562:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829583.720563:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829583.720564:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829583.720565:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787829583.720566:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829583.720568:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1787829583.720569:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829584.614678:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829584.614683:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829584.614725:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829584.614729:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829584.614732:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829585.638703:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829585.638712:0:815:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1787829585.638715:0:815:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1787829585.638718:0:815:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829585.638723:0:815:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1787829585.638726:0:815:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829585.638728:0:815:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829585.638731:0:815:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829585.638781:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829585.638784:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829585.638787:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829586.662781:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829586.662788:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829586.662846:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829586.662850:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829586.662853:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829587.686721:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829587.686728:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829587.686798:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829587.686802:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829587.686806:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829588.710706:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829588.710714:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829588.710823:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829588.710826:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829588.710828:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787829588.838745:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829588.838756:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1787829588.838759:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787829588.838763:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787829588.838768:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787829588.838775:0:468:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 4389 00000100:00080000:3.0:1787829588.838786:0:468:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787829588.838789:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829588.838792:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:3.0:1787829588.838795:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1787829588.838798:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1787829588.838802:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:3.0:1787829588.838804:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:3.0:1787829588.838807:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829588.838808:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:3.0:1787829588.838813:0:468:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:3.0:1787829588.838816:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829588.838819:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787829588.838823:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829588.838826:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829588.838829:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:3.0:1787829588.838832:0:468:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829588.838842:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787829588.838844:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829588.838847:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787829588.838849:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787829588.838854:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829588.838866:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829588.838876:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c7016d00 x1874670646844800/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787829588.838887:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787829588.838891:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829588.838910:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787829588.838914:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829588.838917:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1787829588.838919:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787829588.838921:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787829588.838924:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787829588.838927:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4389 00000100:00080000:3.0:1787829588.838932:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787829588.838934:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829588.838936:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:3.0:1787829588.838939:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1787829588.838941:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1787829588.838944:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000020:00000001:3.0:1787829588.838947:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:3.0:1787829588.838949:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829588.838950:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:3.0:1787829588.838954:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:1.0:1787829588.838958:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1787829588.838958:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829588.838963:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1787829588.838965:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 02000000:00000001:3.0:1787829588.838969:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829588.838970:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787829588.838974:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 02000000:00000001:3.0:1787829588.838975:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829588.838978:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829588.838980:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:3.0:1787829588.838985:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 02000000:00000001:1.0:1787829588.838986:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000008:00000001:3.0:1787829588.838988:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829588.838992:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829588.838992:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829588.838997:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1787829588.838998:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829588.838999:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829588.839015:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787829588.839019:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787829588.839022:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829588.839026:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c7016d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646844800:192.168.203.143@tcp:250:kworker.0 00000100:00000001:3.0:1787829588.839028:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829588.839036:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787829588.839040:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000100:00100000:3.0:1787829588.839043:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c70170c0 x1874670646844928/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:1.0:1787829588.839044:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829588.839050:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:3.0:1787829588.839057:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 02000000:00000001:1.0:1787829588.839060:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829588.839060:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829588.839066:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787829588.839069:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829588.839072:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1787829588.839073:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787829588.839074:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787829588.839076:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787829588.839077:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4389 00000100:00000001:1.0:1787829588.839081:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00080000:3.0:1787829588.839081:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787829588.839085:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787829588.839087:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369547008 : -126234340004608 : ffff8d30c7016d00) 00000100:00000001:3.0:1787829588.839087:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:3.0:1787829588.839093:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1787829588.839095:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:1.0:1787829588.839096:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000020:00000001:3.0:1787829588.839097:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000020:00000001:3.0:1787829588.839101:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:3.0:1787829588.839102:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829588.839104:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:3.0:1787829588.839106:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:3.0:1787829588.839108:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829588.839109:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787829588.839112:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829588.839114:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829588.839115:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:3.0:1787829588.839117:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1787829588.839118:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829588.839119:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829588.839122:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787829588.839124:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829588.839125:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787829588.839126:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787829588.839128:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829588.839131:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829588.839134:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c7017840 x1874670646845056/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787829588.839143:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:3.0:1787829588.839145:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787829588.839147:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829588.839149:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787829588.839151:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829588.839155:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1787829588.839157:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787829588.839157:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787829588.839162:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787829588.839165:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829588.839166:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:1.0:1787829588.839170:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787829588.839172:0:468:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4389 00000100:00000001:1.0:1787829588.839175:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00080000:3.0:1787829588.839179:0:468:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1787829588.839184:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829588.839184:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829588.839186:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:3.0:1787829588.839190:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:1.0:1787829588.839191:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4299063040 00000020:00000001:3.0:1787829588.839193:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1787829588.839197:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000020:00000001:3.0:1787829588.839201:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:3.0:1787829588.839204:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787829588.839206:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 00000100:00000001:3.0:1787829588.839206:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:3.0:1787829588.839213:0:468:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:1.0:1787829588.839217:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4299063040 00000100:00000001:3.0:1787829588.839217:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829588.839220:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1787829588.839225:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 02000000:00000001:3.0:1787829588.839226:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829588.839232:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829588.839233:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829588.839234:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:3.0:1787829588.839238:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00000001:1.0:1787829588.839240:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:3.0:1787829588.839243:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829588.839245:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00000001:3.0:1787829588.839247:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829588.839253:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00100000:1.0:1787829588.839255:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829588/real 1787829588] req@ffff8d30c7016d00 x1874670646844800/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829613 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1787829588.839256:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829588.839261:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787829588.839264:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787829588.839267:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829588.839274:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:1.0:1787829588.839277:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787829588.839282:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00100000:3.0:1787829588.839283:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c7014000 x1874670646845184/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787829588.839287:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787829588.839292:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787829588.839296:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829588.839297:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787829588.839301:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829588.839302:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829588.839306:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829588.839306:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829588.839319:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c7016d00 x1874670646844800/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829613 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787829588.839328:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829588.839333:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787829588.839338:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829588.839340:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829588.839341:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829588.839344:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787829588.839346:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787829588.839349:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829588.839354:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787829588.839356:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787829588.839360:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c7016d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646844800:192.168.203.143@tcp:250:kworker.0 00000100:00000001:1.0:1787829588.839366:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829588.839368:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829588.839369:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787829588.839372:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787829588.839376:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787829588.839378:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787829588.839381:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787829588.839385:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787829588.839387:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787829588.839390:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787829588.839391:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829588.839394:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829588.839414:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787829588.839417:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829588.839419:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787829588.839420:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1787829588.839422:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787829588.839423:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787829588.839425:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829588.839427:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787829588.839429:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829588.839431:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829588.839435:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c70170c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646844928:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787829588.839439:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787829588.839441:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787829588.839442:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829588.839444:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787829588.839447:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829588.839453:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787829588.839455:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369547968 : -126234340003648 : ffff8d30c70170c0) 00000100:00000001:1.0:1787829588.839459:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1787829588.839486:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787829588.839496:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787829588.839499:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829588.839501:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829588.839503:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787829588.839506:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829588.839510:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4299061782 00000100:00000001:1.0:1787829588.839514:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4299063040 00000100:00000001:1.0:1787829588.839516:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4299061782 00000100:00000001:1.0:1787829588.839518:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4299063040 00000100:00000001:1.0:1787829588.839521:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829588.839523:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829588.839525:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787829588.839530:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829588/real 1787829588] req@ffff8d30c70170c0 x1874670646844928/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829613 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787829588.839540:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787829588.839541:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787829588.839544:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787829588.839546:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787829588.839547:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829588.839549:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829588.839551:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787829588.839555:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c70170c0 x1874670646844928/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829613 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787829588.839563:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829588.839565:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787829588.839568:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829588.839570:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829588.839571:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829588.839573:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787829588.839574:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787829588.839577:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829588.839581:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787829588.839583:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787829588.839587:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c70170c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646844928:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787829588.839591:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787829588.839592:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787829588.839593:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829588.839596:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787829588.839598:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829588.839623:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829588.839627:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c7017840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646845056:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787829588.839644:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787829588.839646:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787829588.839648:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829588.839649:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787829588.839653:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829588.839658:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787829588.839660:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369549888 : -126234340001728 : ffff8d30c7017840) 00000100:00000001:1.0:1787829588.839664:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1787829588.839678:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787829588.839683:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787829588.839685:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829588.839687:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829588.839689:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787829588.839691:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829588.839695:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4299061783 00000100:00000001:1.0:1787829588.839697:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4299063040 00000100:00000001:1.0:1787829588.839700:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4299061783 00000100:00000001:1.0:1787829588.839702:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4299063040 00000100:00000001:1.0:1787829588.839704:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829588.839718:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829588.839721:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787829588.839725:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829588/real 1787829588] req@ffff8d30c7017840 x1874670646845056/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829613 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787829588.839734:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787829588.839736:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787829588.839739:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787829588.839741:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787829588.839742:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829588.839744:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829588.839746:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787829588.839750:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c7017840 x1874670646845056/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829613 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787829588.839764:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829588.839767:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787829588.839769:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829588.839771:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829588.839772:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829588.839774:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787829588.839776:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787829588.839779:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829588.839783:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787829588.839786:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787829588.839789:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c7017840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646845056:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787829588.839794:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787829588.839795:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787829588.839796:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829588.839798:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787829588.839800:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829588.839803:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829588.839806:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c7014000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646845184:192.168.203.143@tcp:8:kworker.0 00000100:00000001:1.0:1787829588.839809:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787829588.839811:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787829588.839813:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829588.839815:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787829588.839818:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829588.839824:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787829588.839826:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369535488 : -126234340016128 : ffff8d30c7014000) 00000100:00000001:1.0:1787829588.839829:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1787829588.839842:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787829588.839847:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787829588.839850:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829588.839851:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829588.839853:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787829588.839856:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829588.839860:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4299061796 00000100:00000001:1.0:1787829588.839863:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4299063040 00000100:00000001:1.0:1787829588.839865:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4299061796 00000100:00000001:1.0:1787829588.839867:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4299063040 00000100:00000001:1.0:1787829588.839870:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829588.839872:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829588.839874:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787829588.839878:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829588/real 1787829588] req@ffff8d30c7014000 x1874670646845184/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829613 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787829588.839887:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787829588.839888:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787829588.839891:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787829588.839893:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787829588.839894:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829588.839896:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829588.839898:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787829588.839902:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c7014000 x1874670646845184/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829613 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787829588.839909:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829588.839911:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787829588.839913:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829588.839915:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829588.839916:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829588.839918:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787829588.839920:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787829588.839923:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829588.839926:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787829588.839928:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787829588.839932:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c7014000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646845184:192.168.203.143@tcp:8:kworker.0 00000100:00000001:1.0:1787829588.839936:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829588.839938:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829588.839940:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787829588.839941:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787829588.839944:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787829588.839945:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787829588.839947:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787829588.839949:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787829588.839950:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787829588.839952:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787829588.839954:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829588.839956:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829588.839957:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787829588.839958:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787829588.839960:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787829588.839961:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787829588.839962:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787829588.839964:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787829588.839965:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787829588.839967:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787829588.839968:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829588.839970:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829588.839971:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787829588.839972:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787829588.839974:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787829588.839975:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787829588.839976:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787829588.839978:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787829588.839979:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787829588.839981:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787829588.839982:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829588.839984:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829588.839988:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787829588.839990:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829588.839992:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787829588.839994:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829589.734701:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829589.734708:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829589.734773:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829589.734776:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829589.734779:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829590.758718:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829590.758725:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829590.758806:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829590.758810:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829590.758814:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829591.782718:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829591.782725:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829591.782756:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829591.782761:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829591.782765:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829592.806687:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829592.806693:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829592.806738:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829592.806741:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829592.806743:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829593.830749:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829593.830764:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829593.830838:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829593.830843:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829593.830846:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787829593.958700:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829593.958710:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1787829593.958713:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787829593.958716:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787829593.958721:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787829593.958728:0:468:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 4394 00000100:00080000:3.0:1787829593.958738:0:468:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787829593.958741:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829593.958742:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:3.0:1787829593.958746:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1787829593.958748:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1787829593.958751:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:3.0:1787829593.958753:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:3.0:1787829593.958755:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829593.958756:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:3.0:1787829593.958761:0:468:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:3.0:1787829593.958763:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829593.958765:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787829593.958769:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829593.958771:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829593.958773:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:3.0:1787829593.958776:0:468:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829593.958785:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787829593.958786:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829593.958790:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787829593.958792:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787829593.958796:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829593.958805:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829593.958813:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c7016940 x1874670646845312/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787829593.958830:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787829593.958835:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829593.958849:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787829593.958852:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829593.958857:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1787829593.958858:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787829593.958860:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787829593.958862:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787829593.958864:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4394 00000100:00080000:3.0:1787829593.958868:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787829593.958870:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829593.958871:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:3.0:1787829593.958873:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1787829593.958874:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1787829593.958877:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000020:00000001:3.0:1787829593.958879:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:3.0:1787829593.958882:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829593.958883:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:3.0:1787829593.958886:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:3.0:1787829593.958889:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829593.958891:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787829593.958896:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829593.958899:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829593.958900:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:3.0:1787829593.958904:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1787829593.958907:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829593.958909:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829593.958912:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:3.0:1787829593.958914:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787829593.958919:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829593.958921:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787829593.958922:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:1.0:1787829593.958925:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 02000000:00000001:3.0:1787829593.958926:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:1.0:1787829593.958929:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 02000000:00000001:3.0:1787829593.958931:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829593.958938:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829593.958939:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829593.958949:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c7017c00 x1874670646845440/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:1.0:1787829593.958950:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787829593.958956:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829593.958977:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829593.958992:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c7016940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646845312:192.168.203.143@tcp:250:kworker.0 00000100:00000001:3.0:1787829593.958997:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787829593.959003:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:3.0:1787829593.959003:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829593.959010:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829593.959011:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000100:00080000:3.0:1787829593.959014:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 02000000:00000001:1.0:1787829593.959018:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787829593.959019:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1787829593.959024:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 02000000:00000001:1.0:1787829593.959025:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00080000:3.0:1787829593.959027:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787829593.959031:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787829593.959035:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4394 02000000:00000001:1.0:1787829593.959037:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787829593.959040:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787829593.959045:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829593.959046:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:3.0:1787829593.959051:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1787829593.959052:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1787829593.959054:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000020:00000001:3.0:1787829593.959057:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:3.0:1787829593.959059:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787829593.959062:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787829593.959062:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:1.0:1787829593.959068:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369546048 : -126234340005568 : ffff8d30c7016940) 00000100:00080000:3.0:1787829593.959073:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:1.0:1787829593.959077:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:3.0:1787829593.959078:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829593.959083:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787829593.959087:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829593.959089:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829593.959091:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:3.0:1787829593.959093:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1787829593.959104:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829593.959107:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829593.959114:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787829593.959115:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829593.959118:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787829593.959119:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787829593.959121:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829593.959127:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:1.0:1787829593.959129:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00100000:3.0:1787829593.959138:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c7014000 x1874670646845568/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787829593.959148:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787829593.959149:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787829593.959153:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829593.959155:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829593.959157:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829593.959158:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829593.959168:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00080000:3.0:1787829593.959170:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829593.959176:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1787829593.959177:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829593.959178:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787829593.959183:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787829593.959184:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4299068160 00000100:00000001:3.0:1787829593.959192:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787829593.959197:0:468:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4394 00000100:00000001:1.0:1787829593.959199:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 00000100:00080000:3.0:1787829593.959204:0:468:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787829593.959208:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829593.959210:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:3.0:1787829593.959212:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1787829593.959214:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1787829593.959215:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000020:00000001:3.0:1787829593.959218:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:1.0:1787829593.959219:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4299068160 00000100:00000001:3.0:1787829593.959221:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787829593.959224:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 00000100:00000001:3.0:1787829593.959225:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:1.0:1787829593.959228:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829593.959232:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829593.959236:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00080000:3.0:1787829593.959241:0:468:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00100000:1.0:1787829593.959243:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829593/real 1787829593] req@ffff8d30c7016940 x1874670646845312/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829618 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829593.959251:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829593.959259:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000400:00000001:1.0:1787829593.959266:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 02000000:00000001:3.0:1787829593.959267:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000001:1.0:1787829593.959269:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787829593.959273:0:2390:0:(events.c:78:reply_in_callback()) Process entered 02000000:00000001:3.0:1787829593.959274:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829593.959277:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829593.959279:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:1.0:1787829593.959280:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829593.959285:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000008:00100000:3.0:1787829593.959286:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00000001:1.0:1787829593.959288:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787829593.959298:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c7016940 x1874670646845312/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829618 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00000001:3.0:1787829593.959299:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829593.959310:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829593.959313:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829593.959318:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787829593.959320:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:3.0:1787829593.959320:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:1.0:1787829593.959325:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 02000000:00000001:3.0:1787829593.959326:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829593.959329:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829593.959331:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:1.0:1787829593.959335:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 02000000:00000001:3.0:1787829593.959336:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:1.0:1787829593.959342:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 02000000:00000001:3.0:1787829593.959342:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1787829593.959353:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00000001:3.0:1787829593.959353:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829593.959368:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c7017840 x1874670646845696/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1787829593.959369:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787829593.959379:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000001:3.0:1787829593.959389:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00100000:1.0:1787829593.959390:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c7016940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646845312:192.168.203.143@tcp:250:kworker.0 00000100:00000001:3.0:1787829593.959396:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829593.959399:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829593.959404:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829593.959404:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829593.959407:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787829593.959411:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787829593.959415:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787829593.959417:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787829593.959419:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787829593.959424:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787829593.959425:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787829593.959428:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787829593.959429:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829593.959431:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829593.959439:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787829593.959441:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829593.959443:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787829593.959445:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1787829593.959446:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787829593.959448:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787829593.959450:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829593.959452:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787829593.959455:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829593.959457:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829593.959460:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c7017c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646845440:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787829593.959463:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787829593.959465:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787829593.959466:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829593.959467:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787829593.959470:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829593.959475:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787829593.959476:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369550848 : -126234340000768 : ffff8d30c7017c00) 00000100:00000001:1.0:1787829593.959480:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1787829593.959490:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787829593.959494:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787829593.959495:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829593.959496:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829593.959497:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787829593.959499:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829593.959503:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4299066902 00000100:00000001:1.0:1787829593.959505:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4299068160 00000100:00000001:1.0:1787829593.959507:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4299066902 00000100:00000001:1.0:1787829593.959508:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4299068160 00000100:00000001:1.0:1787829593.959510:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829593.959511:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829593.959512:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787829593.959515:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829593/real 1787829593] req@ffff8d30c7017c00 x1874670646845440/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829618 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787829593.959522:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787829593.959523:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787829593.959525:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787829593.959526:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787829593.959527:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829593.959528:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829593.959529:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787829593.959532:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c7017c00 x1874670646845440/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829618 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787829593.959536:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829593.959538:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787829593.959539:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829593.959540:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829593.959541:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829593.959542:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787829593.959543:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787829593.959545:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829593.959547:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787829593.959548:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787829593.959550:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c7017c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646845440:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787829593.959553:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787829593.959554:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787829593.959555:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829593.959557:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787829593.959558:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829593.959559:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829593.959561:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c7014000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646845568:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787829593.959563:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787829593.959565:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787829593.959566:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829593.959567:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787829593.959570:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829593.959573:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787829593.959574:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369535488 : -126234340016128 : ffff8d30c7014000) 00000100:00000001:1.0:1787829593.959576:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1787829593.959585:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787829593.959589:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787829593.959591:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829593.959592:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829593.959594:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787829593.959596:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829593.959639:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4299066903 00000100:00000001:1.0:1787829593.959643:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4299068160 00000100:00000001:1.0:1787829593.959645:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4299066903 00000100:00000001:1.0:1787829593.959648:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4299068160 00000100:00000001:1.0:1787829593.959650:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829593.959653:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829593.959655:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787829593.959660:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829593/real 1787829593] req@ffff8d30c7014000 x1874670646845568/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829618 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787829593.959668:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787829593.959670:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787829593.959673:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787829593.959675:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787829593.959676:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829593.959678:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829593.959689:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787829593.959694:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c7014000 x1874670646845568/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829618 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787829593.959701:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829593.959703:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787829593.959706:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829593.959707:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829593.959709:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829593.959710:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787829593.959712:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787829593.959714:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829593.959717:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787829593.959719:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787829593.959721:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c7014000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646845568:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787829593.959725:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787829593.959726:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787829593.959727:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829593.959730:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787829593.959731:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829593.959733:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829593.959736:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c7017840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646845696:192.168.203.143@tcp:8:kworker.0 00000100:00000001:1.0:1787829593.959739:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787829593.959741:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787829593.959742:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829593.959744:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787829593.959747:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829593.959753:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787829593.959754:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369549888 : -126234340001728 : ffff8d30c7017840) 00000100:00000001:1.0:1787829593.959757:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1787829593.959768:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787829593.959771:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787829593.959773:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829593.959775:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829593.959777:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787829593.959779:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829593.959783:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4299066916 00000100:00000001:1.0:1787829593.959786:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4299068160 00000100:00000001:1.0:1787829593.959788:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4299066916 00000100:00000001:1.0:1787829593.959790:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4299068160 00000100:00000001:1.0:1787829593.959793:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829593.959795:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829593.959797:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787829593.959801:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829593/real 1787829593] req@ffff8d30c7017840 x1874670646845696/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829618 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787829593.959810:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787829593.959811:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787829593.959814:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787829593.959816:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787829593.959817:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829593.959819:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829593.959821:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787829593.959825:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c7017840 x1874670646845696/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829618 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787829593.959830:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829593.959831:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787829593.959833:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829593.959834:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829593.959834:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829593.959836:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787829593.959837:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787829593.959840:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829593.959843:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787829593.959845:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787829593.959848:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c7017840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646845696:192.168.203.143@tcp:8:kworker.0 00000100:00000001:1.0:1787829593.959853:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829593.959855:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829593.959857:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787829593.959858:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787829593.959861:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787829593.959862:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787829593.959864:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787829593.959866:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787829593.959868:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787829593.959870:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787829593.959871:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829593.959873:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829593.959875:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787829593.959876:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787829593.959879:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787829593.959881:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787829593.959882:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787829593.959884:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787829593.959885:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787829593.959887:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787829593.959889:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829593.959891:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829593.959892:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787829593.959894:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787829593.959896:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787829593.959897:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787829593.959898:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787829593.959900:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787829593.959901:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787829593.959903:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787829593.959904:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829593.959906:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829593.959909:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787829593.959911:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829593.959913:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787829593.959915:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829594.854680:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829594.854686:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829594.854733:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829594.854736:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829594.854739:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829595.878755:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829595.878766:0:815:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1787829595.878768:0:815:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1787829595.878771:0:815:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829595.878776:0:815:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1787829595.878778:0:815:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829595.878780:0:815:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829595.878781:0:815:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829595.878896:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829595.878902:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829595.878907:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829596.902719:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829596.902728:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829596.902785:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829596.902789:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829596.902792:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829597.926674:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829597.926678:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829597.926726:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829597.926730:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829597.926734:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829598.950754:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829598.950759:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829598.950906:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829598.950910:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829598.950914:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787829599.078707:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829599.078716:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1787829599.078718:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787829599.078726:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787829599.078730:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787829599.078738:0:468:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 4399 00000100:00080000:3.0:1787829599.078749:0:468:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787829599.078752:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829599.078754:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:3.0:1787829599.078759:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1787829599.078761:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1787829599.078764:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:3.0:1787829599.078766:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:3.0:1787829599.078767:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829599.078769:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:3.0:1787829599.078772:0:468:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:3.0:1787829599.078774:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829599.078776:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787829599.078779:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829599.078781:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829599.078782:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:3.0:1787829599.078784:0:468:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829599.078795:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787829599.078796:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829599.078799:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787829599.078801:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787829599.078805:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829599.078815:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829599.078824:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c70170c0 x1874670646845824/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787829599.078832:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787829599.078835:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829599.078847:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787829599.078849:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829599.078851:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1787829599.078852:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787829599.078853:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787829599.078855:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787829599.078856:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4399 00000100:00080000:3.0:1787829599.078859:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787829599.078860:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829599.078861:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:3.0:1787829599.078863:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1787829599.078864:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1787829599.078865:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000020:00000001:3.0:1787829599.078867:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:3.0:1787829599.078868:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829599.078869:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:3.0:1787829599.078871:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:3.0:1787829599.078873:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829599.078875:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787829599.078879:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829599.078881:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829599.078882:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:1.0:1787829599.078886:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000008:00100000:3.0:1787829599.078886:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1787829599.078891:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829599.078893:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787829599.078893:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829599.078897:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 02000000:00000001:3.0:1787829599.078898:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787829599.078901:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829599.078903:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787829599.078903:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787829599.078906:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:1.0:1787829599.078907:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829599.078908:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829599.078914:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829599.078915:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000100:00100000:3.0:1787829599.078919:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c7016d00 x1874670646845952/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:1.0:1787829599.078920:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829599.078928:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787829599.078930:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829599.078941:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829599.078946:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829599.078950:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c70170c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646845824:192.168.203.143@tcp:250:kworker.0 00000100:00080000:3.0:1787829599.078951:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829599.078955:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1787829599.078958:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:1.0:1787829599.078960:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00080000:3.0:1787829599.078960:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787829599.078964:0:468:0:(import.c:485:import_select_connection()) Process entered 02000000:00000001:1.0:1787829599.078966:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000100:00080000:3.0:1787829599.078967:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4399 02000000:00000001:1.0:1787829599.078971:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787829599.078972:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787829599.078976:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 02000000:00000001:1.0:1787829599.078977:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:3.0:1787829599.078977:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:3.0:1787829599.078981:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1787829599.078981:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 02000000:00000001:1.0:1787829599.078983:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829599.078983:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000020:00000001:3.0:1787829599.078987:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:3.0:1787829599.078991:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829599.078993:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:3.0:1787829599.078996:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:1.0:1787829599.078998:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787829599.078999:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829599.079000:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369547968 : -126234340003648 : ffff8d30c70170c0) 02000000:00000001:3.0:1787829599.079003:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1787829599.079005:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 02000000:00000001:3.0:1787829599.079008:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829599.079012:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829599.079014:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:3.0:1787829599.079016:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1787829599.079019:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829599.079021:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829599.079025:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787829599.079027:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829599.079028:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787829599.079030:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787829599.079031:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829599.079035:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829599.079039:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c7014b40 x1874670646846080/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1787829599.079044:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:3.0:1787829599.079048:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787829599.079050:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829599.079052:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787829599.079054:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829599.079057:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1787829599.079058:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787829599.079058:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787829599.079060:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787829599.079062:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787829599.079064:0:468:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4399 00000100:00080000:3.0:1787829599.079067:0:468:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787829599.079070:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829599.079072:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:1.0:1787829599.079073:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000020:00000001:3.0:1787829599.079075:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:1.0:1787829599.079077:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829599.079078:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:1.0:1787829599.079081:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000020:00000001:3.0:1787829599.079082:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000020:00000001:3.0:1787829599.079086:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:1.0:1787829599.079089:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829599.079089:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829599.079091:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:3.0:1787829599.079094:0:468:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:3.0:1787829599.079099:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829599.079100:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4299073280 02000000:00000001:3.0:1787829599.079101:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787829599.079107:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829599.079109:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829599.079111:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:1.0:1787829599.079112:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 00000008:00100000:3.0:1787829599.079114:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1787829599.079118:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829599.079120:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829599.079124:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787829599.079125:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829599.079131:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4299073280 00000100:00000001:3.0:1787829599.079137:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:1.0:1787829599.079138:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 02000000:00000001:3.0:1787829599.079140:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:1.0:1787829599.079143:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829599.079144:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829599.079149:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829599.079153:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00000001:3.0:1787829599.079154:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829599.079160:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829599/real 1787829599] req@ffff8d30c70170c0 x1874670646845824/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829624 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1787829599.079163:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c70152c0 x1874670646846208/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787829599.079179:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787829599.079181:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829599.079183:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:1.0:1787829599.079187:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787829599.079190:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787829599.079194:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787829599.079197:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787829599.079199:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829599.079202:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829599.079204:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787829599.079210:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c70170c0 x1874670646845824/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829624 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787829599.079218:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829599.079220:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787829599.079223:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829599.079225:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829599.079226:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829599.079228:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787829599.079229:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787829599.079232:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829599.079235:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787829599.079237:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787829599.079241:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c70170c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646845824:192.168.203.143@tcp:250:kworker.0 00000100:00000001:1.0:1787829599.079246:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829599.079248:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829599.079250:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787829599.079252:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787829599.079256:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787829599.079257:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787829599.079259:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787829599.079263:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787829599.079264:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787829599.079266:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787829599.079267:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829599.079270:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829599.079276:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787829599.079278:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829599.079280:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787829599.079282:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1787829599.079283:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787829599.079285:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787829599.079286:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829599.079288:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787829599.079290:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829599.079292:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829599.079295:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c7016d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646845952:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787829599.079298:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787829599.079300:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787829599.079302:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829599.079303:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787829599.079306:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829599.079311:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787829599.079313:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369547008 : -126234340004608 : ffff8d30c7016d00) 00000100:00000001:1.0:1787829599.079317:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1787829599.079328:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787829599.079333:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787829599.079335:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829599.079336:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829599.079337:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787829599.079340:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829599.079344:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4299072022 00000100:00000001:1.0:1787829599.079347:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4299073280 00000100:00000001:1.0:1787829599.079349:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4299072022 00000100:00000001:1.0:1787829599.079351:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4299073280 00000100:00000001:1.0:1787829599.079353:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829599.079355:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829599.079356:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787829599.079361:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829599/real 1787829599] req@ffff8d30c7016d00 x1874670646845952/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829624 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787829599.079369:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787829599.079371:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787829599.079373:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787829599.079375:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787829599.079376:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829599.079378:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829599.079379:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787829599.079383:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c7016d00 x1874670646845952/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829624 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787829599.079387:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829599.079389:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787829599.079391:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829599.079393:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829599.079394:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829599.079396:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787829599.079398:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787829599.079400:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829599.079403:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787829599.079405:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787829599.079408:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c7016d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646845952:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787829599.079412:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787829599.079413:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787829599.079415:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829599.079417:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787829599.079418:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829599.079420:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829599.079423:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c7014b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646846080:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787829599.079426:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787829599.079428:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787829599.079429:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829599.079431:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787829599.079434:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829599.079439:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787829599.079440:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369538368 : -126234340013248 : ffff8d30c7014b40) 00000100:00000001:1.0:1787829599.079452:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1787829599.079465:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787829599.079469:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787829599.079472:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829599.079473:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829599.079475:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787829599.079477:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829599.079480:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4299072022 00000100:00000001:1.0:1787829599.079481:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4299073280 00000100:00000001:1.0:1787829599.079483:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4299072022 00000100:00000001:1.0:1787829599.079484:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4299073280 00000100:00000001:1.0:1787829599.079485:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829599.079486:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829599.079487:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787829599.079490:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829599/real 1787829599] req@ffff8d30c7014b40 x1874670646846080/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829624 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787829599.079495:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787829599.079495:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787829599.079497:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787829599.079498:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787829599.079499:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829599.079500:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829599.079501:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787829599.079503:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c7014b40 x1874670646846080/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829624 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787829599.079507:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829599.079509:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787829599.079510:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829599.079511:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829599.079511:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829599.079512:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787829599.079513:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787829599.079514:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829599.079517:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787829599.079518:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787829599.079520:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c7014b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646846080:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787829599.079522:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787829599.079523:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787829599.079524:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829599.079525:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787829599.079526:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829599.079527:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829599.079528:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c70152c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646846208:192.168.203.143@tcp:8:kworker.0 00000100:00000001:1.0:1787829599.079530:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787829599.079532:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787829599.079532:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829599.079533:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787829599.079535:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829599.079538:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787829599.079539:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369540288 : -126234340011328 : ffff8d30c70152c0) 00000100:00000001:1.0:1787829599.079541:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1787829599.079548:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787829599.079550:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787829599.079551:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829599.079552:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829599.079553:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787829599.079555:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829599.079556:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4299072035 00000100:00000001:1.0:1787829599.079558:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4299073280 00000100:00000001:1.0:1787829599.079559:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4299072035 00000100:00000001:1.0:1787829599.079561:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4299073280 00000100:00000001:1.0:1787829599.079562:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829599.079563:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829599.079564:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787829599.079566:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829599/real 1787829599] req@ffff8d30c70152c0 x1874670646846208/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829624 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787829599.079571:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787829599.079572:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787829599.079573:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787829599.079573:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787829599.079574:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829599.079575:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829599.079576:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787829599.079578:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c70152c0 x1874670646846208/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829624 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787829599.079582:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829599.079583:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787829599.079584:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829599.079585:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829599.079585:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829599.079586:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787829599.079587:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787829599.079588:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829599.079590:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787829599.079591:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787829599.079592:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c70152c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646846208:192.168.203.143@tcp:8:kworker.0 00000100:00000001:1.0:1787829599.079594:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829599.079595:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829599.079596:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787829599.079597:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787829599.079598:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787829599.079599:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787829599.079621:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787829599.079623:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787829599.079624:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787829599.079625:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787829599.079626:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829599.079628:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829599.079628:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787829599.079629:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787829599.079630:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787829599.079631:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787829599.079632:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787829599.079634:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787829599.079635:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787829599.079636:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787829599.079637:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829599.079639:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829599.079639:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787829599.079641:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787829599.079642:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787829599.079644:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787829599.079644:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787829599.079646:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787829599.079647:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787829599.079648:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787829599.079654:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829599.079657:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829599.079660:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787829599.079661:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829599.079663:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787829599.079665:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829599.974656:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829599.974661:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829599.974713:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829599.974717:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829599.974721:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829600.998661:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829600.998667:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829600.998712:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829600.998716:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829600.998720:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829602.022655:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829602.022661:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829602.022699:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829602.022701:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829602.022704:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829603.046664:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829603.046668:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829603.046719:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829603.046722:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829603.046725:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829604.070659:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829604.070664:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829604.070724:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829604.070729:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829604.070732:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787829604.198747:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829604.198757:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1787829604.198760:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787829604.198763:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787829604.198768:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787829604.198776:0:468:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 4404 00000100:00080000:3.0:1787829604.198787:0:468:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787829604.198791:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829604.198794:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:3.0:1787829604.198799:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1787829604.198802:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1787829604.198805:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:3.0:1787829604.198809:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:3.0:1787829604.198811:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829604.198813:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:3.0:1787829604.198819:0:468:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:3.0:1787829604.198823:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829604.198826:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787829604.198830:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829604.198834:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829604.198836:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:3.0:1787829604.198839:0:468:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829604.198851:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787829604.198853:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829604.198857:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787829604.198859:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787829604.198863:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829604.198875:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829604.198884:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c7015680 x1874670646846336/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787829604.198909:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787829604.198915:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829604.198930:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787829604.198933:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829604.198936:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1787829604.198938:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787829604.198940:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787829604.198943:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787829604.198946:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4404 00000100:00080000:3.0:1787829604.198952:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787829604.198954:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829604.198956:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:3.0:1787829604.198959:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:1.0:1787829604.198961:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:3.0:1787829604.198961:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1787829604.198965:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:1.0:1787829604.198967:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1787829604.198971:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000020:00000001:3.0:1787829604.198972:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:1.0:1787829604.198974:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787829604.198978:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829604.198980:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:1.0:1787829604.198983:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787829604.198984:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 02000000:00000001:1.0:1787829604.198988:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:3.0:1787829604.198989:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829604.198991:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829604.198994:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1787829604.198996:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829604.199002:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c7015680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646846336:192.168.203.143@tcp:250:kworker.0 02000000:00000001:3.0:1787829604.199003:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829604.199008:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1787829604.199009:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829604.199013:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 02000000:00000001:1.0:1787829604.199014:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787829604.199019:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000008:00100000:3.0:1787829604.199019:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 02000000:00000001:1.0:1787829604.199024:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000008:00000001:3.0:1787829604.199025:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829604.199029:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829604.199031:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829604.199034:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787829604.199037:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829604.199040:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787829604.199042:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787829604.199044:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829604.199048:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787829604.199051:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829604.199052:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369541248 : -126234340010368 : ffff8d30c7015680) 00000100:00000001:1.0:1787829604.199059:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00100000:3.0:1787829604.199060:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c70143c0 x1874670646846464/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787829604.199072:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787829604.199074:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829604.199078:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787829604.199081:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829604.199084:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1787829604.199087:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787829604.199088:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787829604.199091:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787829604.199094:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4404 00000100:00080000:3.0:1787829604.199099:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787829604.199102:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829604.199104:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000800:00000100:1.0:1787829604.199107:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000020:00000001:3.0:1787829604.199108:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1787829604.199112:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1787829604.199114:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000020:00000001:3.0:1787829604.199116:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:3.0:1787829604.199118:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829604.199120:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:3.0:1787829604.199124:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:3.0:1787829604.199127:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829604.199129:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787829604.199132:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829604.199134:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829604.199136:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:3.0:1787829604.199138:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1787829604.199141:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829604.199143:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829604.199147:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787829604.199149:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829604.199151:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787829604.199153:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787829604.199155:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829604.199161:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829604.199164:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00100000:3.0:1787829604.199169:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c7014f00 x1874670646846592/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787829604.199170:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829604.199173:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829604.199177:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787829604.199181:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829604.199184:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787829604.199186:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4299078400 00000100:00000001:3.0:1787829604.199189:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829604.199195:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829604.199196:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 00000100:00080000:3.0:1787829604.199198:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829604.199203:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1787829604.199205:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:1.0:1787829604.199207:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4299078400 00000100:00080000:3.0:1787829604.199207:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787829604.199214:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 00000100:00000001:3.0:1787829604.199214:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:1.0:1787829604.199220:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00080000:3.0:1787829604.199221:0:468:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4404 00000100:00000001:1.0:1787829604.199223:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829604.199228:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00080000:3.0:1787829604.199230:0:468:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787829604.199234:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00100000:1.0:1787829604.199235:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829604/real 1787829604] req@ffff8d30c7015680 x1874670646846336/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829629 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829604.199236:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:3.0:1787829604.199243:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1787829604.199247:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1787829604.199251:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000400:00000001:1.0:1787829604.199253:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787829604.199255:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000020:00000001:3.0:1787829604.199257:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:3.0:1787829604.199262:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829604.199263:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:3.0:1787829604.199268:0:468:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:1.0:1787829604.199270:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829604.199271:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829604.199274:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 02000000:00000001:3.0:1787829604.199276:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1787829604.199277:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829604.199280:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 02000000:00000001:3.0:1787829604.199282:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829604.199283:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:3.0:1787829604.199287:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829604.199290:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c7015680 x1874670646846336/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829629 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787829604.199296:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829604.199299:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:1.0:1787829604.199301:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000008:00100000:3.0:1787829604.199303:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00000001:1.0:1787829604.199305:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000008:00000001:3.0:1787829604.199307:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829604.199308:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829604.199311:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829604.199312:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829604.199317:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787829604.199319:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 02000000:00000001:3.0:1787829604.199319:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00080000:1.0:1787829604.199322:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 02000000:00000001:3.0:1787829604.199323:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829604.199328:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00080000:1.0:1787829604.199331:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 02000000:00000001:3.0:1787829604.199331:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787829604.199335:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829604.199336:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787829604.199341:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c7015680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646846336:192.168.203.143@tcp:250:kworker.0 00000100:00000001:3.0:1787829604.199341:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829604.199349:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829604.199351:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c70161c0 x1874670646846720/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787829604.199352:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829604.199355:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787829604.199358:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787829604.199361:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00000001:3.0:1787829604.199362:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000020:00000001:1.0:1787829604.199364:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000100:00000001:3.0:1787829604.199365:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829604.199367:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 00000100:00000001:3.0:1787829604.199369:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829604.199372:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787829604.199375:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787829604.199382:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787829604.199384:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829604.199386:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829604.199393:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787829604.199395:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829604.199397:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787829604.199399:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1787829604.199400:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787829604.199401:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787829604.199402:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829604.199404:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787829604.199406:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829604.199407:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829604.199410:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c70143c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646846464:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787829604.199412:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787829604.199414:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787829604.199415:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829604.199416:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787829604.199418:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829604.199423:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787829604.199424:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369536448 : -126234340015168 : ffff8d30c70143c0) 00000100:00000001:1.0:1787829604.199427:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1787829604.199437:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787829604.199441:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787829604.199442:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829604.199443:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829604.199444:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787829604.199446:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829604.199449:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4299077142 00000100:00000001:1.0:1787829604.199451:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4299078400 00000100:00000001:1.0:1787829604.199453:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4299077142 00000100:00000001:1.0:1787829604.199454:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4299078400 00000100:00000001:1.0:1787829604.199456:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829604.199457:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829604.199458:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787829604.199461:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829604/real 1787829604] req@ffff8d30c70143c0 x1874670646846464/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829629 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787829604.199466:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787829604.199467:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787829604.199469:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787829604.199470:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787829604.199471:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829604.199472:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829604.199473:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787829604.199476:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c70143c0 x1874670646846464/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829629 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787829604.199481:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829604.199482:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787829604.199483:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829604.199484:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829604.199485:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829604.199486:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787829604.199487:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787829604.199489:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829604.199491:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787829604.199492:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787829604.199494:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c70143c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646846464:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787829604.199497:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787829604.199497:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787829604.199499:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829604.199500:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787829604.199501:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829604.199502:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829604.199504:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c7014f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646846592:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787829604.199506:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787829604.199507:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787829604.199508:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829604.199509:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787829604.199511:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829604.199514:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787829604.199515:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369539328 : -126234340012288 : ffff8d30c7014f00) 00000100:00000001:1.0:1787829604.199517:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1787829604.199529:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787829604.199532:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787829604.199534:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829604.199535:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829604.199536:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787829604.199538:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829604.199540:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4299077142 00000100:00000001:1.0:1787829604.199541:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4299078400 00000100:00000001:1.0:1787829604.199543:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4299077142 00000100:00000001:1.0:1787829604.199544:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4299078400 00000100:00000001:1.0:1787829604.199546:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829604.199547:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829604.199548:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787829604.199551:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829604/real 1787829604] req@ffff8d30c7014f00 x1874670646846592/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829629 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787829604.199556:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787829604.199556:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787829604.199558:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787829604.199558:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787829604.199559:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829604.199560:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829604.199561:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787829604.199563:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c7014f00 x1874670646846592/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829629 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787829604.199568:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829604.199569:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787829604.199571:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829604.199572:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829604.199572:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829604.199573:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787829604.199574:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787829604.199576:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829604.199578:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787829604.199579:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787829604.199582:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c7014f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646846592:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787829604.199584:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787829604.199585:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787829604.199586:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829604.199587:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787829604.199588:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829604.199589:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829604.199591:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c70161c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646846720:192.168.203.143@tcp:8:kworker.0 00000100:00000001:1.0:1787829604.199593:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787829604.199594:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787829604.199595:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829604.199596:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787829604.199598:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829604.199633:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787829604.199635:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369544128 : -126234340007488 : ffff8d30c70161c0) 00000100:00000001:1.0:1787829604.199637:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1787829604.199645:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787829604.199648:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787829604.199649:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829604.199650:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829604.199651:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787829604.199653:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829604.199655:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4299077156 00000100:00000001:1.0:1787829604.199657:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4299078400 00000100:00000001:1.0:1787829604.199658:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4299077156 00000100:00000001:1.0:1787829604.199660:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4299078400 00000100:00000001:1.0:1787829604.199661:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829604.199662:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829604.199663:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787829604.199666:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829604/real 1787829604] req@ffff8d30c70161c0 x1874670646846720/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829629 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787829604.199676:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787829604.199677:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787829604.199678:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787829604.199679:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787829604.199680:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829604.199681:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829604.199682:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787829604.199684:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c70161c0 x1874670646846720/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829629 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787829604.199689:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829604.199690:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787829604.199692:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829604.199693:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829604.199693:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829604.199695:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787829604.199696:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787829604.199699:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829604.199702:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787829604.199704:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787829604.199707:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c70161c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646846720:192.168.203.143@tcp:8:kworker.0 00000100:00000001:1.0:1787829604.199711:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829604.199713:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829604.199714:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787829604.199716:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787829604.199718:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787829604.199720:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787829604.199721:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787829604.199723:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787829604.199724:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787829604.199726:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787829604.199727:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829604.199730:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829604.199731:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787829604.199732:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787829604.199735:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787829604.199736:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787829604.199737:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787829604.199739:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787829604.199740:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787829604.199742:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787829604.199743:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829604.199745:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829604.199746:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787829604.199748:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787829604.199750:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787829604.199751:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787829604.199752:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787829604.199754:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787829604.199755:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787829604.199756:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787829604.199757:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829604.199758:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829604.199761:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787829604.199762:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829604.199763:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787829604.199764:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829605.094680:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829605.094687:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829605.094735:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829605.094739:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829605.094742:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829606.118739:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829606.118749:0:815:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1787829606.118752:0:815:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1787829606.118755:0:815:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829606.118760:0:815:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1787829606.118762:0:815:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829606.118764:0:815:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829606.118766:0:815:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829606.118814:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829606.118819:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829606.118823:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829607.142692:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829607.142698:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829607.142759:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829607.142763:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829607.142767:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829608.166744:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829608.166751:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829608.166822:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829608.166828:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829608.166832:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829609.190724:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829609.190732:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829609.190833:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829609.190838:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829609.190841:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787829609.318733:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829609.318743:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1787829609.318746:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787829609.318750:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787829609.318755:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787829609.318763:0:468:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 4409 00000100:00080000:3.0:1787829609.318774:0:468:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787829609.318777:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829609.318779:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:3.0:1787829609.318783:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1787829609.318786:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1787829609.318789:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:3.0:1787829609.318791:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:3.0:1787829609.318792:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829609.318794:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:3.0:1787829609.318798:0:468:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:3.0:1787829609.318800:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829609.318802:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787829609.318806:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829609.318808:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829609.318810:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:3.0:1787829609.318812:0:468:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829609.318825:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787829609.318827:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829609.318830:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787829609.318832:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787829609.318837:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829609.318850:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829609.318859:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c7015a40 x1874670646846848/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787829609.318882:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787829609.318886:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829609.318902:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787829609.318905:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829609.318907:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1787829609.318909:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787829609.318910:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787829609.318912:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787829609.318914:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4409 00000100:00080000:3.0:1787829609.318916:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787829609.318918:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829609.318919:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:3.0:1787829609.318921:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1787829609.318923:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1787829609.318925:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000020:00000001:3.0:1787829609.318927:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:3.0:1787829609.318929:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829609.318931:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:3.0:1787829609.318933:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:1.0:1787829609.318935:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1787829609.318936:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829609.318940:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1787829609.318942:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1787829609.318945:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 02000000:00000001:3.0:1787829609.318946:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829609.318948:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 02000000:00000001:3.0:1787829609.318949:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829609.318952:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829609.318952:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 02000000:00000001:1.0:1787829609.318959:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787829609.318961:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000008:00100000:3.0:1787829609.318963:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 02000000:00000001:1.0:1787829609.318964:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:3.0:1787829609.318970:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829609.318972:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829609.318977:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c7015a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646846848:192.168.203.143@tcp:250:kworker.0 02000000:00000001:3.0:1787829609.318977:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787829609.318979:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829609.318982:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787829609.318985:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:1.0:1787829609.318988:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1787829609.318988:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829609.318994:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829609.318996:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787829609.319001:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829609.319004:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c7016580 x1874670646846976/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:1.0:1787829609.319006:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787829609.319014:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829609.319016:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787829609.319020:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829609.319024:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787829609.319027:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829609.319031:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1787829609.319033:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787829609.319035:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787829609.319037:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787829609.319039:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:1.0:1787829609.319041:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369542208 : -126234340009408 : ffff8d30c7015a40) 00000100:00080000:3.0:1787829609.319044:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4409 00000100:00000001:1.0:1787829609.319050:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00080000:3.0:1787829609.319052:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787829609.319055:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829609.319056:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:3.0:1787829609.319059:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1787829609.319060:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1787829609.319062:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000020:00000001:3.0:1787829609.319065:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:3.0:1787829609.319067:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829609.319069:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:3.0:1787829609.319073:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:3.0:1787829609.319077:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829609.319079:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787829609.319082:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829609.319085:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829609.319088:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:3.0:1787829609.319090:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1787829609.319093:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829609.319095:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:1.0:1787829609.319103:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 02000000:00000001:3.0:1787829609.319103:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787829609.319106:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829609.319110:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787829609.319118:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787829609.319121:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829609.319123:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787829609.319126:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829609.319130:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00100000:3.0:1787829609.319133:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c70161c0 x1874670646847104/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787829609.319135:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829609.319141:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1787829609.319145:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787829609.319147:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829609.319150:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829609.319151:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787829609.319152:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829609.319159:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1787829609.319160:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4299083520 00000100:00000001:3.0:1787829609.319162:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787829609.319166:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787829609.319176:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 00000100:00000001:3.0:1787829609.319177:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787829609.319184:0:468:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4409 00000100:00080000:3.0:1787829609.319190:0:468:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787829609.319192:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829609.319194:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:3.0:1787829609.319196:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1787829609.319198:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:1.0:1787829609.319200:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4299083520 00000020:00000001:3.0:1787829609.319201:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000020:00000001:3.0:1787829609.319207:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:3.0:1787829609.319210:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787829609.319212:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 00000100:00000001:3.0:1787829609.319212:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:3.0:1787829609.319219:0:468:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:1.0:1787829609.319220:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829609.319222:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829609.319226:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1787829609.319227:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829609.319229:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829609.319233:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 02000000:00000001:3.0:1787829609.319233:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829609.319237:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:3.0:1787829609.319240:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1787829609.319243:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829609.319244:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829609/real 1787829609] req@ffff8d30c7015a40 x1874670646846848/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829634 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829609.319245:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829609.319251:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787829609.319253:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:1.0:1787829609.319261:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787829609.319264:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829609.319264:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787829609.319268:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:1.0:1787829609.319269:0:2390:0:(events.c:78:reply_in_callback()) Process entered 02000000:00000001:3.0:1787829609.319272:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829609.319273:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787829609.319277:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829609.319280:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829609.319281:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829609.319282:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829609.319293:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c7014f00 x1874670646847232/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787829609.319299:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c7015a40 x1874670646846848/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829634 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829609.319325:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787829609.319326:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829609.319329:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829609.319332:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829609.319333:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829609.319338:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829609.319340:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829609.319342:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829609.319344:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787829609.319346:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787829609.319348:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829609.319351:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787829609.319353:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787829609.319357:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c7015a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646846848:192.168.203.143@tcp:250:kworker.0 00000100:00000001:1.0:1787829609.319367:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829609.319369:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829609.319371:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787829609.319373:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787829609.319378:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787829609.319380:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787829609.319382:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787829609.319385:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787829609.319387:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787829609.319390:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787829609.319392:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829609.319394:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829609.319413:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787829609.319415:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829609.319417:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787829609.319419:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1787829609.319423:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787829609.319424:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787829609.319426:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829609.319428:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787829609.319430:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829609.319432:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829609.319436:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c7016580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646846976:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787829609.319439:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787829609.319441:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787829609.319443:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829609.319445:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787829609.319448:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829609.319454:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787829609.319456:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369545088 : -126234340006528 : ffff8d30c7016580) 00000100:00000001:1.0:1787829609.319462:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1787829609.319477:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787829609.319482:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787829609.319484:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829609.319486:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829609.319495:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787829609.319499:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829609.319504:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4299082262 00000100:00000001:1.0:1787829609.319506:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4299083520 00000100:00000001:1.0:1787829609.319508:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4299082262 00000100:00000001:1.0:1787829609.319511:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4299083520 00000100:00000001:1.0:1787829609.319513:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829609.319515:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829609.319516:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787829609.319521:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829609/real 1787829609] req@ffff8d30c7016580 x1874670646846976/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829634 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787829609.319530:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787829609.319531:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787829609.319534:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787829609.319535:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787829609.319537:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829609.319538:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829609.319540:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787829609.319544:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c7016580 x1874670646846976/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829634 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787829609.319557:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829609.319559:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787829609.319562:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829609.319564:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829609.319565:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829609.319567:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787829609.319569:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787829609.319571:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829609.319575:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787829609.319578:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787829609.319582:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c7016580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646846976:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787829609.319586:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787829609.319588:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787829609.319589:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829609.319592:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787829609.319593:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829609.319595:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829609.319597:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c70161c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646847104:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787829609.319599:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787829609.319622:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787829609.319623:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829609.319625:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787829609.319628:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829609.319632:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787829609.319633:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369544128 : -126234340007488 : ffff8d30c70161c0) 00000100:00000001:1.0:1787829609.319636:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1787829609.319646:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787829609.319649:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787829609.319651:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829609.319652:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829609.319653:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787829609.319655:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829609.319658:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4299082263 00000100:00000001:1.0:1787829609.319661:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4299083520 00000100:00000001:1.0:1787829609.319664:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4299082263 00000100:00000001:1.0:1787829609.319666:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4299083520 00000100:00000001:1.0:1787829609.319669:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829609.319671:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829609.319673:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787829609.319678:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829609/real 1787829609] req@ffff8d30c70161c0 x1874670646847104/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829634 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787829609.319687:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787829609.319688:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787829609.319690:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787829609.319692:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787829609.319693:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829609.319695:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829609.319697:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787829609.319701:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c70161c0 x1874670646847104/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829634 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787829609.319709:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829609.319711:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787829609.319713:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829609.319715:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829609.319716:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829609.319719:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787829609.319720:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787829609.319723:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829609.319727:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787829609.319729:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787829609.319733:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c70161c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646847104:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787829609.319738:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787829609.319740:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787829609.319741:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829609.319743:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787829609.319745:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829609.319748:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829609.319751:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c7014f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646847232:192.168.203.143@tcp:8:kworker.0 00000100:00000001:1.0:1787829609.319755:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787829609.319757:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787829609.319759:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829609.319761:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787829609.319764:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829609.319771:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787829609.319772:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369539328 : -126234340012288 : ffff8d30c7014f00) 00000100:00000001:1.0:1787829609.319776:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1787829609.319790:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787829609.319795:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787829609.319797:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829609.319798:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829609.319800:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787829609.319803:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829609.319807:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4299082276 00000100:00000001:1.0:1787829609.319810:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4299083520 00000100:00000001:1.0:1787829609.319812:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4299082276 00000100:00000001:1.0:1787829609.319814:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4299083520 00000100:00000001:1.0:1787829609.319817:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829609.319819:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829609.319821:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787829609.319826:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829609/real 1787829609] req@ffff8d30c7014f00 x1874670646847232/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829634 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787829609.319842:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787829609.319843:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787829609.319846:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787829609.319847:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787829609.319849:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829609.319851:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829609.319853:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787829609.319857:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c7014f00 x1874670646847232/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829634 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787829609.319880:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829609.319882:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787829609.319885:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829609.319887:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829609.319888:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829609.319890:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787829609.319892:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787829609.319895:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829609.319899:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787829609.319901:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787829609.319905:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c7014f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646847232:192.168.203.143@tcp:8:kworker.0 00000100:00000001:1.0:1787829609.319909:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829609.319911:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829609.319913:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787829609.319915:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787829609.319918:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787829609.319919:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787829609.319921:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787829609.319923:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787829609.319925:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787829609.319927:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787829609.319928:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829609.319930:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829609.319932:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787829609.319933:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787829609.319935:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787829609.319936:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787829609.319938:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787829609.319939:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787829609.319941:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787829609.319942:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787829609.319944:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829609.319946:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829609.319947:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787829609.319948:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787829609.319951:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787829609.319952:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787829609.319954:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787829609.319956:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787829609.319957:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787829609.319959:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787829609.319960:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829609.319962:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829609.319966:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787829609.319968:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829609.319970:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787829609.319972:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829610.214713:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829610.214723:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829610.214774:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829610.214778:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829610.214781:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829611.238669:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829611.238675:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829611.238719:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829611.238723:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829611.238726:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829612.262701:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829612.262708:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829612.262759:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829612.262762:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829612.262765:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829613.286700:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829613.286706:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829613.286750:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829613.286755:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829613.286758:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829614.310722:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829614.310727:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829614.310770:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829614.310774:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829614.310777:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787829614.438685:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829614.438693:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1787829614.438696:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1787829614.438699:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787829614.438703:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1787829614.438718:0:468:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 4414 00000100:00080000:1.0:1787829614.438727:0:468:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1787829614.438730:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787829614.438731:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:1.0:1787829614.438735:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:1.0:1787829614.438738:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:1.0:1787829614.438740:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:1.0:1787829614.438742:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:1.0:1787829614.438744:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787829614.438746:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:1.0:1787829614.438751:0:468:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:1.0:1787829614.438754:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829614.438757:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1787829614.438761:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829614.438764:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829614.438767:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:1.0:1787829614.438770:0:468:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829614.438779:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1787829614.438781:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829614.438784:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1787829614.438787:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1787829614.438792:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829614.438803:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829614.438814:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d13ee940 x1874670646847360/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787829614.438837:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787829614.438843:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829614.438858:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1787829614.438861:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829614.438865:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1787829614.438868:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1787829614.438869:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787829614.438874:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1787829614.438878:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4414 00000100:00080000:1.0:1787829614.438884:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787829614.438884:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787829614.438886:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787829614.438889:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:3.0:1787829614.438890:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000020:00000001:1.0:1787829614.438892:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:3.0:1787829614.438893:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000020:00000001:1.0:1787829614.438894:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:1.0:1787829614.438898:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:3.0:1787829614.438899:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787829614.438902:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829614.438903:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:1.0:1787829614.438908:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 02000000:00000001:3.0:1787829614.438908:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787829614.438912:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829614.438913:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 02000000:00000001:3.0:1787829614.438917:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1787829614.438922:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00100000:3.0:1787829614.438924:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d13ee940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646847360:192.168.203.143@tcp:250:kworker.0 00000100:00000001:1.0:1787829614.438927:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829614.438930:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787829614.438932:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787829614.438934:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1787829614.438937:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829614.438940:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829614.438940:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787829614.438946:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829614.438946:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829614.438953:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:1.0:1787829614.438959:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1787829614.438963:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829614.438964:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787829614.438965:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829614.438967:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839541348672 : -126234168202944 : ffff8d30d13ee940) 00000100:00000001:3.0:1787829614.438972:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 02000000:00000001:1.0:1787829614.438975:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1787829614.438978:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829614.438981:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1787829614.438982:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1787829614.438985:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829614.438992:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829614.438999:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d13ec3c0 x1874670646847488/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1787829614.439010:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787829614.439014:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787829614.439017:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829614.439021:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1787829614.439024:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829614.439028:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:1.0:1787829614.439029:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1787829614.439031:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787829614.439034:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1787829614.439037:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4414 00000100:00080000:1.0:1787829614.439041:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1787829614.439043:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787829614.439044:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:1.0:1787829614.439046:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:1.0:1787829614.439047:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:1.0:1787829614.439049:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000020:00000001:1.0:1787829614.439052:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:1.0:1787829614.439054:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787829614.439056:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:1.0:1787829614.439059:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:1.0:1787829614.439061:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829614.439063:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1787829614.439065:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829614.439067:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829614.439070:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:1.0:1787829614.439071:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1787829614.439074:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829614.439076:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829614.439080:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1787829614.439081:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829614.439083:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1787829614.439085:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1787829614.439087:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829614.439091:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829614.439097:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d13ec780 x1874670646847616/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787829614.439106:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787829614.439112:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829614.439114:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787829614.439115:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829614.439119:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787829614.439120:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829614.439125:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829614.439126:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829614.439130:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4299088640 00000100:00080000:1.0:1787829614.439131:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829614.439138:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1787829614.439140:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1787829614.439142:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787829614.439145:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1787829614.439148:0:468:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4414 00000100:00080000:1.0:1787829614.439152:0:468:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1787829614.439155:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787829614.439156:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:1.0:1787829614.439159:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:1.0:1787829614.439160:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:1.0:1787829614.439162:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000020:00000001:1.0:1787829614.439165:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:1.0:1787829614.439167:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787829614.439169:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:1.0:1787829614.439173:0:468:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:1.0:1787829614.439176:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829614.439178:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1787829614.439181:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829614.439183:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829614.439186:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:1.0:1787829614.439188:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000008:00000001:1.0:1787829614.439191:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829614.439193:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829614.439197:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1787829614.439199:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829614.439201:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1787829614.439203:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1787829614.439205:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829614.439209:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829614.439214:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d13ec000 x1874670646847744/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787829614.439222:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787829614.439224:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829614.439226:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829614.439312:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 00000100:00000001:3.0:1787829614.439320:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4299088640 00000100:00000001:3.0:1787829614.439328:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 00000100:00000001:3.0:1787829614.439331:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829614.439334:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829614.439338:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1787829614.439398:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829614/real 1787829614] req@ffff8d30d13ee940 x1874670646847360/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829639 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1787829614.439421:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787829614.439423:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829614.439427:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829614.439430:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829614.439431:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829614.439434:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829614.439437:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829614.439442:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d13ee940 x1874670646847360/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829639 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829614.439451:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829614.439454:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829614.439456:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829614.439458:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829614.439459:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829614.439460:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829614.439462:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829614.439464:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829614.439467:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829614.439469:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829614.439471:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d13ee940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646847360:192.168.203.143@tcp:250:kworker.0 00000100:00000001:3.0:1787829614.439475:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829614.439477:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829614.439478:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829614.439480:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829614.439482:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829614.439484:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829614.439485:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829614.439488:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829614.439489:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829614.439491:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829614.439492:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829614.439494:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829614.439500:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787829614.439502:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829614.439503:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1787829614.439504:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787829614.439505:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787829614.439506:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787829614.439507:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829614.439509:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787829614.439510:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829614.439512:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829614.439514:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d13ec3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646847488:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829614.439516:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1787829614.439518:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1787829614.439519:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829614.439520:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1787829614.439523:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829614.439528:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787829614.439529:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839541339072 : -126234168212544 : ffff8d30d13ec3c0) 00000100:00000001:3.0:1787829614.439532:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1787829614.439544:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:3.0:1787829614.439547:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787829614.439549:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829614.439550:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829614.439551:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1787829614.439554:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829614.439557:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4299087382 00000100:00000001:3.0:1787829614.439559:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4299088640 00000100:00000001:3.0:1787829614.439560:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4299087382 00000100:00000001:3.0:1787829614.439562:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4299088640 00000100:00000001:3.0:1787829614.439563:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829614.439564:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829614.439566:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1787829614.439568:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829614/real 1787829614] req@ffff8d30d13ec3c0 x1874670646847488/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829639 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1787829614.439579:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787829614.439580:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829614.439582:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829614.439584:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829614.439585:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829614.439587:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829614.439588:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829614.439592:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d13ec3c0 x1874670646847488/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829639 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829614.439627:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829614.439631:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829614.439634:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829614.439637:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829614.439639:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829614.439641:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829614.439643:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829614.439647:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829614.439651:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829614.439654:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829614.439658:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d13ec3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646847488:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829614.439661:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787829614.439662:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787829614.439663:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829614.439665:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787829614.439666:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829614.439668:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829614.439670:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d13ec780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646847616:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829614.439672:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1787829614.439674:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1787829614.439676:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829614.439678:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1787829614.439681:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829614.439686:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787829614.439687:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839541340032 : -126234168211584 : ffff8d30d13ec780) 00000100:00000001:3.0:1787829614.439690:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1787829614.439703:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:3.0:1787829614.439707:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787829614.439709:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829614.439710:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829614.439711:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1787829614.439713:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829614.439715:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4299087383 00000100:00000001:3.0:1787829614.439717:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4299088640 00000100:00000001:3.0:1787829614.439719:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4299087383 00000100:00000001:3.0:1787829614.439721:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4299088640 00000100:00000001:3.0:1787829614.439723:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829614.439724:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829614.439726:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1787829614.439729:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829614/real 1787829614] req@ffff8d30d13ec780 x1874670646847616/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829639 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1787829614.439734:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787829614.439734:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829614.439736:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829614.439737:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829614.439738:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829614.439739:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829614.439740:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829614.439742:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d13ec780 x1874670646847616/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829639 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829614.439749:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829614.439751:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829614.439754:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829614.439756:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829614.439757:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829614.439759:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829614.439761:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829614.439764:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829614.439767:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829614.439770:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829614.439773:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d13ec780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646847616:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829614.439777:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787829614.439779:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787829614.439780:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829614.439782:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787829614.439784:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829614.439786:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829614.439789:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d13ec000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646847744:192.168.203.143@tcp:8:kworker.0 00000100:00000001:3.0:1787829614.439793:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1787829614.439795:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1787829614.439797:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829614.439799:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1787829614.439802:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829614.439808:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787829614.439810:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839541338112 : -126234168213504 : ffff8d30d13ec000) 00000100:00000001:3.0:1787829614.439813:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1787829614.439823:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:3.0:1787829614.439840:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787829614.439842:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829614.439843:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829614.439845:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1787829614.439847:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829614.439850:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4299087396 00000100:00000001:3.0:1787829614.439853:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4299088640 00000100:00000001:3.0:1787829614.439856:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4299087396 00000100:00000001:3.0:1787829614.439858:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4299088640 00000100:00000001:3.0:1787829614.439860:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829614.439862:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829614.439865:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1787829614.439869:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829614/real 1787829614] req@ffff8d30d13ec000 x1874670646847744/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829639 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1787829614.439878:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787829614.439880:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829614.439882:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829614.439884:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829614.439886:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829614.439888:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829614.439889:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829614.439893:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d13ec000 x1874670646847744/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829639 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829614.439901:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829614.439903:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829614.439905:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829614.439907:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829614.439908:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829614.439910:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829614.439912:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829614.439915:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829614.439918:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829614.439921:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829614.439924:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d13ec000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646847744:192.168.203.143@tcp:8:kworker.0 00000100:00000001:3.0:1787829614.439928:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829614.439930:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829614.439932:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829614.439934:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829614.439936:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829614.439938:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829614.439939:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829614.439942:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829614.439944:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829614.439946:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829614.439947:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829614.439949:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829614.439951:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829614.439952:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829614.439954:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829614.439955:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829614.439956:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829614.439958:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829614.439959:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829614.439960:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829614.439962:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829614.439963:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829614.439964:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829614.439966:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829614.439968:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829614.439969:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829614.439970:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829614.439972:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829614.439973:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829614.439974:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829614.439975:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829614.439977:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829614.439979:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787829614.439981:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829614.439982:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1787829614.439983:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829615.334717:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829615.334723:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829615.334782:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829615.334787:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829615.334790:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829616.358720:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829616.358728:0:815:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1787829616.358731:0:815:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1787829616.358735:0:815:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829616.358740:0:815:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1787829616.358742:0:815:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829616.358744:0:815:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829616.358747:0:815:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829616.358805:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829616.358809:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829616.358825:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829617.382711:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829617.382718:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829617.382781:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829617.382785:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829617.382787:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829618.406734:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829618.406742:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829618.406839:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829618.406845:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829618.406852:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829619.430669:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829619.430674:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829619.430774:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829619.430779:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829619.430783:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787829619.558708:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829619.558718:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1787829619.558721:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1787829619.558725:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787829619.558730:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1787829619.558737:0:468:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 4420 00000100:00080000:1.0:1787829619.558750:0:468:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1787829619.558753:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787829619.558754:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:1.0:1787829619.558759:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:1.0:1787829619.558761:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:1.0:1787829619.558765:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:1.0:1787829619.558766:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:1.0:1787829619.558769:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787829619.558770:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:1.0:1787829619.558774:0:468:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:1.0:1787829619.558776:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829619.558778:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1787829619.558782:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829619.558784:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829619.558786:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:1.0:1787829619.558788:0:468:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829619.558815:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1787829619.558817:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829619.558820:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1787829619.558822:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1787829619.558827:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829619.558839:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829619.558848:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d13ecb40 x1874670646847872/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787829619.558858:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787829619.558862:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829619.558876:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1787829619.558879:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829619.558881:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1787829619.558883:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1787829619.558884:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787829619.558886:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1787829619.558888:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4420 00000100:00080000:1.0:1787829619.558890:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1787829619.558892:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787829619.558893:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:1.0:1787829619.558895:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:1.0:1787829619.558896:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:1.0:1787829619.558898:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000020:00000001:1.0:1787829619.558901:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:1.0:1787829619.558903:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787829619.558904:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:1.0:1787829619.558907:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:1.0:1787829619.558909:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829619.558910:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:3.0:1787829619.558912:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:1.0:1787829619.558916:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829619.558918:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829619.558920:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:3.0:1787829619.558920:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787829619.558926:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000008:00100000:1.0:1787829619.558927:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1787829619.558931:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829619.558931:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787829619.558935:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829619.558937:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829619.558939:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1787829619.558941:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829619.558944:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1787829619.558945:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787829619.558946:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787829619.558947:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829619.558953:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829619.558953:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829619.558961:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829619.558963:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d13ef0c0 x1874670646848000/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787829619.558971:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d13ecb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646847872:192.168.203.143@tcp:250:kworker.0 00000100:00000001:1.0:1787829619.558973:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787829619.558976:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829619.558982:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829619.558982:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00080000:1.0:1787829619.558987:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829619.558990:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:1.0:1787829619.558991:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 02000000:00000001:3.0:1787829619.558991:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000100:00080000:1.0:1787829619.558993:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787829619.558996:0:468:0:(import.c:485:import_select_connection()) Process entered 02000000:00000001:3.0:1787829619.558997:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1787829619.558999:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4420 00000100:00080000:1.0:1787829619.559004:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 02000000:00000001:3.0:1787829619.559004:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:1.0:1787829619.559008:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787829619.559010:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:1.0:1787829619.559012:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:1.0:1787829619.559013:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 02000000:00000001:3.0:1787829619.559014:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829619.559015:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000020:00000001:1.0:1787829619.559020:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:1.0:1787829619.559022:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787829619.559023:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:1.0:1787829619.559024:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:1.0:1787829619.559026:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829619.559027:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1787829619.559029:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829619.559031:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829619.559032:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:1.0:1787829619.559033:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1787829619.559035:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829619.559036:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829619.559038:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 02000000:00000001:1.0:1787829619.559039:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1787829619.559043:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829619.559044:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:3.0:1787829619.559044:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839541340992 : -126234168210624 : ffff8d30d13ecb40) 02000000:00000001:1.0:1787829619.559047:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1787829619.559049:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829619.559053:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:1.0:1787829619.559054:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829619.559058:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d13ed680 x1874670646848128/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787829619.559063:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787829619.559065:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829619.559066:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1787829619.559068:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829619.559070:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1787829619.559071:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1787829619.559071:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787829619.559073:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1787829619.559076:0:468:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4420 00000100:00080000:1.0:1787829619.559079:0:468:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1787829619.559080:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787829619.559081:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:1.0:1787829619.559082:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:1.0:1787829619.559083:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:1.0:1787829619.559084:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000020:00000001:1.0:1787829619.559086:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:1.0:1787829619.559087:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787829619.559088:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:1.0:1787829619.559091:0:468:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:1.0:1787829619.559093:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829619.559094:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1787829619.559096:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829619.559097:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829619.559099:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:1.0:1787829619.559100:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000008:00000001:1.0:1787829619.559101:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829619.559103:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829619.559106:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1787829619.559108:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:3.0:1787829619.559108:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787829619.559112:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1787829619.559114:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1787829619.559115:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829619.559120:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829619.559124:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d13ed2c0 x1874670646848256/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787829619.559129:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787829619.559130:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787829619.559131:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829619.559135:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829619.559138:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829619.559143:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829619.559146:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1787829619.559154:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829619.559160:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4299093760 00000100:00000001:3.0:1787829619.559172:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 00000100:00000001:3.0:1787829619.559188:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4299093760 00000100:00000001:3.0:1787829619.559195:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 00000100:00000001:3.0:1787829619.559199:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829619.559202:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829619.559205:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1787829619.559213:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829619/real 1787829619] req@ffff8d30d13ecb40 x1874670646847872/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829644 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1787829619.559226:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787829619.559229:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829619.559233:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829619.559236:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829619.559238:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829619.559241:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829619.559243:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829619.559249:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d13ecb40 x1874670646847872/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829644 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829619.559258:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829619.559261:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829619.559265:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829619.559266:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829619.559268:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829619.559270:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829619.559273:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829619.559276:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829619.559280:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829619.559284:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829619.559288:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d13ecb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646847872:192.168.203.143@tcp:250:kworker.0 00000100:00000001:3.0:1787829619.559293:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829619.559295:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829619.559297:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829619.559300:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829619.559304:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829619.559307:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829619.559309:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829619.559315:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829619.559317:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829619.559320:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829619.559322:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829619.559324:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829619.559334:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787829619.559337:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829619.559339:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1787829619.559340:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787829619.559342:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787829619.559343:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787829619.559345:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829619.559347:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787829619.559349:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829619.559351:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829619.559355:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d13ef0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646848000:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829619.559359:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1787829619.559362:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1787829619.559363:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829619.559365:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1787829619.559368:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829619.559374:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787829619.559376:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839541350592 : -126234168201024 : ffff8d30d13ef0c0) 00000100:00000001:3.0:1787829619.559381:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1787829619.559395:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:3.0:1787829619.559400:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787829619.559403:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829619.559404:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829619.559406:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1787829619.559409:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829619.559413:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4299092502 00000100:00000001:3.0:1787829619.559417:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4299093760 00000100:00000001:3.0:1787829619.559419:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4299092502 00000100:00000001:3.0:1787829619.559421:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4299093760 00000100:00000001:3.0:1787829619.559423:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829619.559426:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829619.559428:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1787829619.559433:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829619/real 1787829619] req@ffff8d30d13ef0c0 x1874670646848000/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829644 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1787829619.559442:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787829619.559443:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829619.559446:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829619.559447:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829619.559449:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829619.559451:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829619.559452:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829619.559456:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d13ef0c0 x1874670646848000/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829644 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829619.559470:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829619.559472:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829619.559475:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829619.559477:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829619.559479:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829619.559481:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829619.559483:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829619.559485:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829619.559489:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829619.559491:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829619.559495:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d13ef0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646848000:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829619.559499:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787829619.559501:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787829619.559502:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829619.559505:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787829619.559507:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829619.559509:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829619.559513:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d13ed680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646848128:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829619.559516:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1787829619.559518:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1787829619.559520:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829619.559522:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1787829619.559525:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829619.559530:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787829619.559532:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839541343872 : -126234168207744 : ffff8d30d13ed680) 00000100:00000001:3.0:1787829619.559535:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1787829619.559547:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:3.0:1787829619.559552:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787829619.559555:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829619.559569:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829619.559576:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1787829619.559580:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829619.559584:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4299092502 00000100:00000001:3.0:1787829619.559587:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4299093760 00000100:00000001:3.0:1787829619.559590:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4299092502 00000100:00000001:3.0:1787829619.559592:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4299093760 00000100:00000001:3.0:1787829619.559594:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829619.559597:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829619.559635:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1787829619.559642:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829619/real 1787829619] req@ffff8d30d13ed680 x1874670646848128/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829644 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1787829619.559651:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787829619.559653:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829619.559656:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829619.559657:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829619.559659:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829619.559661:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829619.559663:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829619.559667:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d13ed680 x1874670646848128/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829644 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829619.559674:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829619.559676:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829619.559679:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829619.559680:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829619.559682:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829619.559683:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829619.559685:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829619.559688:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829619.559691:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829619.559693:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829619.559697:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d13ed680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646848128:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829619.559701:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787829619.559703:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787829619.559704:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829619.559706:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787829619.559708:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829619.559711:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829619.559714:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d13ed2c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646848256:192.168.203.143@tcp:8:kworker.0 00000100:00000001:3.0:1787829619.559718:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1787829619.559720:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1787829619.559722:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829619.559724:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1787829619.559727:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829619.559733:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787829619.559735:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839541342912 : -126234168208704 : ffff8d30d13ed2c0) 00000100:00000001:3.0:1787829619.559738:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1787829619.559751:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:3.0:1787829619.559757:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787829619.559759:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829619.559761:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829619.559763:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1787829619.559766:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829619.559770:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4299092516 00000100:00000001:3.0:1787829619.559773:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4299093760 00000100:00000001:3.0:1787829619.559776:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4299092516 00000100:00000001:3.0:1787829619.559779:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4299093760 00000100:00000001:3.0:1787829619.559782:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829619.559784:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829619.559786:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1787829619.559805:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829619/real 1787829619] req@ffff8d30d13ed2c0 x1874670646848256/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829644 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1787829619.559821:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787829619.559822:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829619.559825:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829619.559827:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829619.559829:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829619.559831:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829619.559832:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829619.559837:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d13ed2c0 x1874670646848256/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829644 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829619.559844:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829619.559846:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829619.559849:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829619.559850:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829619.559852:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829619.559854:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829619.559856:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829619.559858:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829619.559861:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829619.559864:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829619.559868:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d13ed2c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646848256:192.168.203.143@tcp:8:kworker.0 00000100:00000001:3.0:1787829619.559871:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829619.559874:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829619.559875:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829619.559877:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829619.559880:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829619.559882:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829619.559883:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829619.559887:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829619.559888:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829619.559891:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829619.559892:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829619.559895:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829619.559896:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829619.559898:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829619.559900:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829619.559901:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829619.559902:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829619.559904:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829619.559905:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829619.559907:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829619.559909:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829619.559910:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829619.559912:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829619.559913:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829619.559915:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829619.559916:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829619.559918:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829619.559919:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829619.559921:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829619.559923:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829619.559924:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829619.559926:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829619.559929:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787829619.559931:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829619.559933:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1787829619.559935:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829620.454719:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829620.454726:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829620.454841:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829620.454845:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829620.454848:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829621.478673:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829621.478680:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829621.478741:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829621.478744:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829621.478747:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829622.502692:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829622.502697:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829622.502745:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829622.502749:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829622.502751:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829623.526676:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829623.526681:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829623.526738:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829623.526742:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829623.526746:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829624.550691:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829624.550696:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829624.550765:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829624.550769:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829624.550773:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787829624.678686:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829624.678693:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1787829624.678695:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787829624.678699:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787829624.678702:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787829624.678708:0:468:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 4425 00000100:00080000:3.0:1787829624.678716:0:468:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787829624.678719:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829624.678721:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:3.0:1787829624.678724:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1787829624.678726:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1787829624.678728:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:3.0:1787829624.678730:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:3.0:1787829624.678731:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829624.678732:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:3.0:1787829624.678736:0:468:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:3.0:1787829624.678737:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829624.678739:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787829624.678742:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829624.678744:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829624.678745:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:3.0:1787829624.678747:0:468:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829624.678767:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787829624.678768:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829624.678770:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787829624.678772:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787829624.678776:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829624.678786:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829624.678793:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c70143c0 x1874670646848384/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787829624.678801:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787829624.678803:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829624.678815:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787829624.678817:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829624.678819:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1787829624.678820:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787829624.678821:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787829624.678823:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787829624.678824:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4425 00000100:00080000:3.0:1787829624.678828:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787829624.678830:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829624.678831:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:3.0:1787829624.678833:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1787829624.678834:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1787829624.678836:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000020:00000001:3.0:1787829624.678837:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:3.0:1787829624.678838:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829624.678839:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:3.0:1787829624.678841:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:3.0:1787829624.678843:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829624.678845:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787829624.678849:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829624.678850:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:3.0:1787829624.678851:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829624.678855:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:1.0:1787829624.678858:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000008:00100000:3.0:1787829624.678859:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00000001:1.0:1787829624.678863:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000008:00000001:3.0:1787829624.678863:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829624.678867:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829624.678869:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 02000000:00000001:3.0:1787829624.678870:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:1.0:1787829624.678873:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829624.678873:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829624.678878:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787829624.678880:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1787829624.678882:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787829624.678882:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829624.678887:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829624.678888:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829624.678894:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829624.678896:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c7015680 x1874670646848512/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787829624.678903:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c70143c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646848384:192.168.203.143@tcp:250:kworker.0 00000100:00000001:3.0:1787829624.678904:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787829624.678907:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829624.678911:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829624.678914:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00080000:3.0:1787829624.678914:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829624.678919:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 02000000:00000001:1.0:1787829624.678920:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:3.0:1787829624.678920:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787829624.678923:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 02000000:00000001:1.0:1787829624.678925:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829624.678926:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787829624.678930:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4425 02000000:00000001:1.0:1787829624.678931:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00080000:3.0:1787829624.678933:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787829624.678935:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829624.678936:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:3.0:1787829624.678937:0:468:0:(genops.c:792:class_conn2export()) Process entered 02000000:00000001:1.0:1787829624.678938:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829624.678939:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1787829624.678941:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000020:00000001:3.0:1787829624.678944:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:3.0:1787829624.678945:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829624.678946:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:3.0:1787829624.678948:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:3.0:1787829624.678950:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829624.678951:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787829624.678954:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829624.678956:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 02000000:00000001:3.0:1787829624.678956:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829624.678960:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369536448 : -126234340015168 : ffff8d30c70143c0) 00000100:00000001:3.0:1787829624.678961:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:3.0:1787829624.678963:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1787829624.678966:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829624.678967:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:3.0:1787829624.678969:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829624.678975:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787829624.678977:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829624.678979:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787829624.678980:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787829624.678982:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829624.678988:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829624.678993:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c70152c0 x1874670646848640/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787829624.679002:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787829624.679005:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829624.679008:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787829624.679011:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1787829624.679014:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787829624.679015:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1787829624.679019:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:1.0:1787829624.679021:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4299098880 00000100:00080000:3.0:1787829624.679022:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787829624.679027:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787829624.679030:0:468:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4425 00000100:00000001:1.0:1787829624.679033:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 00000100:00080000:3.0:1787829624.679035:0:468:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787829624.679038:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829624.679039:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:1.0:1787829624.679042:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4299098880 00000020:00000001:3.0:1787829624.679042:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1787829624.679045:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1787829624.679048:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:1.0:1787829624.679050:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 00000020:00000001:3.0:1787829624.679051:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:3.0:1787829624.679054:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829624.679057:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:1.0:1787829624.679059:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00080000:3.0:1787829624.679060:0:468:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:1.0:1787829624.679066:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829624.679067:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829624.679071:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1787829624.679074:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829624.679075:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829624.679080:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:3.0:1787829624.679080:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829624.679081:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:3.0:1787829624.679085:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1787829624.679087:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829624.679089:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829624.679090:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 02000000:00000001:3.0:1787829624.679093:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:1.0:1787829624.679094:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 02000000:00000001:3.0:1787829624.679095:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829624.679099:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787829624.679101:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:1.0:1787829624.679102:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:3.0:1787829624.679103:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829624.679106:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1787829624.679109:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787829624.679109:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829624.679114:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00100000:3.0:1787829624.679117:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c7014b40 x1874670646848768/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787829624.679118:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829624.679124:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787829624.679128:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829624.679128:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787829624.679132:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829624.679134:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829624.679136:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829624.679142:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c7015680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646848512:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787829624.679147:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787829624.679149:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787829624.679152:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829624.679154:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787829624.679166:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829624.679174:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787829624.679177:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369541248 : -126234340010368 : ffff8d30c7015680) 00000100:00000001:1.0:1787829624.679181:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:1.0:1787829624.679194:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829624.679198:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4299097622 00000100:00000001:1.0:1787829624.679201:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4299098880 00000100:00000001:1.0:1787829624.679203:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4299097622 00000100:00000001:1.0:1787829624.679206:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4299098880 00000100:00000001:1.0:1787829624.679208:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829624.679210:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829624.679212:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787829624.679213:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787829624.679215:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829624.679217:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787829624.679219:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829624.679221:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829624.679224:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c70152c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646848640:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787829624.679228:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787829624.679229:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787829624.679231:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829624.679233:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787829624.679236:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829624.679242:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787829624.679243:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369540288 : -126234340011328 : ffff8d30c70152c0) 00000100:00000001:1.0:1787829624.679246:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:1.0:1787829624.679255:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829624.679259:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4299097622 00000100:00000001:1.0:1787829624.679261:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4299098880 00000100:00000001:1.0:1787829624.679263:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4299097622 00000100:00000001:1.0:1787829624.679265:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4299098880 00000100:00000001:1.0:1787829624.679267:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829624.679269:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829624.679272:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829624.679274:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829624.679277:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787829624.679278:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:1.0:1787829624.679280:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787829624.679282:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1787829624.679283:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787829624.679284:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787829624.679286:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829624.679288:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787829624.679290:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829624.679292:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829624.679296:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c7014b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646848768:192.168.203.143@tcp:8:kworker.0 00000100:00000001:1.0:1787829624.679299:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787829624.679301:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787829624.679303:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829624.679304:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787829624.679307:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829624.679312:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787829624.679313:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369538368 : -126234340013248 : ffff8d30c7014b40) 00000100:00000001:1.0:1787829624.679316:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:1.0:1787829624.679326:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829624.679330:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4299097635 00000100:00000001:1.0:1787829624.679333:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4299098880 00000100:00000001:1.0:1787829624.679335:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4299097635 00000100:00000001:1.0:1787829624.679337:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4299098880 00000100:00000001:1.0:1787829624.679339:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829624.679341:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829624.679344:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829624.679346:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829624.679348:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787829624.679350:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:1.0:1787829624.679352:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787829624.679353:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1787829624.679355:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829624.679357:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829625.574676:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829625.574682:0:815:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1787829625.574684:0:815:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1787829625.574686:0:815:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829625.574689:0:815:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1787829625.574691:0:815:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829625.574693:0:815:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829625.574694:0:815:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829625.574770:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829625.574775:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829625.574779:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829626.598687:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829626.598692:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829626.598800:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829626.598804:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829626.598807:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829627.622692:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829627.622700:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829627.622759:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829627.622763:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829627.622767:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:2.0:1787829627.751006:0:2379:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.143:988 00000400:00000100:2.0:1787829627.751072:0:2379:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.143@tcp at host 192.168.203.143 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1787829627.751090:0:2379:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.143@tcp 00000100:00000001:2.0:1787829627.751132:0:2379:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1787829627.751156:0:2379:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787829627.751161:0:2379:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829627.751164:0:2379:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:2.0:1787829627.751178:0:2379:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.143@tcp 00000100:00000001:2.0:1787829627.751185:0:2379:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1787829627.751189:0:2379:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787829627.751191:0:2379:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829627.751194:0:2379:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:2.0:1787829627.751219:0:2379:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.143@tcp 00000100:00000001:2.0:1787829627.751225:0:2379:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1787829627.751230:0:2379:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787829627.751232:0:2379:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829627.751234:0:2379:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:2.0:1787829627.751240:0:2379:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.143@tcp 00000100:00000001:1.0:1787829627.751245:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787829627.751247:0:2379:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787829627.751252:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1787829627.751253:0:2379:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829627.751258:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00000001:2.0:1787829627.751258:0:2379:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829627.751264:0:2379:0:(events.c:68:request_out_callback()) Process leaving 00000100:00100000:1.0:1787829627.751286:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829624/real 1787829627] req@ffff8d30c7014b40 x1874670646848768/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829649 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787829627.751305:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787829627.751307:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787829627.751315:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787829627.751318:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787829627.751321:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829627.751325:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829627.751327:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787829627.751333:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c7014b40 x1874670646848768/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829649 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787829627.751351:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829627.751355:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787829627.751360:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829627.751361:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829627.751370:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829627.751374:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787829627.751377:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787829627.751382:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829627.751387:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787829627.751391:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787829627.751395:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c7014b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646848768:192.168.203.143@tcp:8:kworker.0 00000100:00000001:1.0:1787829627.751401:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787829627.751422:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829624/real 1787829627] req@ffff8d30c7015680 x1874670646848512/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829649 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787829627.751431:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787829627.751433:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787829627.751435:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787829627.751437:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787829627.751438:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829627.751440:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829627.751442:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787829627.751446:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c7015680 x1874670646848512/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829649 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787829627.751454:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829627.751456:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787829627.751459:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829627.751465:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829627.751467:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829627.751469:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787829627.751471:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787829627.751474:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829627.751479:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787829627.751481:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787829627.751485:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c7015680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646848512:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787829627.751489:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787829627.751494:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829624/real 1787829627] req@ffff8d30c70152c0 x1874670646848640/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829649 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787829627.751502:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787829627.751503:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787829627.751506:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787829627.751508:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787829627.751509:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829627.751511:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829627.751512:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787829627.751516:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c70152c0 x1874670646848640/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829649 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787829627.751523:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829627.751525:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787829627.751528:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829627.751529:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829627.751531:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829627.751533:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787829627.751535:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787829627.751538:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829627.751541:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787829627.751544:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787829627.751547:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c70152c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646848640:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787829627.751551:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787829627.751556:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829624/real 1787829627] req@ffff8d30c70143c0 x1874670646848384/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829649 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787829627.751563:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787829627.751565:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787829627.751568:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787829627.751569:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787829627.751571:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829627.751573:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829627.751574:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787829627.751578:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c70143c0 x1874670646848384/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829649 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787829627.751584:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829627.751586:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787829627.751589:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829627.751590:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829627.751592:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829627.751594:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787829627.751596:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787829627.751598:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829627.751630:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787829627.751632:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787829627.751636:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c70143c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646848384:192.168.203.143@tcp:250:kworker.0 00000100:00000001:1.0:1787829627.751640:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829627.751643:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829627.751645:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787829627.751648:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787829627.751654:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787829627.751657:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787829627.751660:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787829627.751665:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787829627.751667:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787829627.751671:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787829627.751672:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829627.751674:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829627.751675:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787829627.751677:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787829627.751679:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787829627.751680:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787829627.751682:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787829627.751684:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787829627.751685:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787829627.751688:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787829627.751689:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829627.751691:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829627.751692:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787829627.751693:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787829627.751695:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787829627.751696:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787829627.751697:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787829627.751699:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787829627.751700:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787829627.751702:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787829627.751703:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829627.751705:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829627.751706:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787829627.751707:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787829627.751709:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787829627.751710:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787829627.751711:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787829627.751713:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787829627.751714:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787829627.751716:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787829627.751717:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829627.751719:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829627.751727:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787829627.751730:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829627.751744:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787829627.751746:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829628.646682:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829628.646688:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829628.646748:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829628.646751:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829628.646754:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829629.670689:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829629.670695:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829629.670753:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829629.670758:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829629.670762:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787829629.798698:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829629.798709:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1787829629.798712:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787829629.798715:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787829629.798733:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787829629.798740:0:468:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 4430 00000100:00080000:3.0:1787829629.798751:0:468:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787829629.798754:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829629.798756:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:3.0:1787829629.798760:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1787829629.798763:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1787829629.798766:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:3.0:1787829629.798769:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:3.0:1787829629.798771:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829629.798773:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:3.0:1787829629.798777:0:468:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:3.0:1787829629.798780:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829629.798783:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787829629.798786:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829629.798789:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829629.798792:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:3.0:1787829629.798796:0:468:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829629.798804:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787829629.798806:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829629.798810:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787829629.798813:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787829629.798817:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829629.798827:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829629.798836:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c7016d00 x1874670646848896/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787829629.798846:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787829629.798851:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829629.798874:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787829629.798877:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829629.798881:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1787829629.798882:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787829629.798884:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787829629.798887:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787829629.798890:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4430 00000100:00080000:3.0:1787829629.798894:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787829629.798896:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829629.798897:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:3.0:1787829629.798900:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1787829629.798901:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1787829629.798903:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000020:00000001:3.0:1787829629.798905:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:3.0:1787829629.798907:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829629.798909:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:3.0:1787829629.798912:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:3.0:1787829629.798915:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829629.798917:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787829629.798923:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829629.798926:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829629.798928:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:3.0:1787829629.798933:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1787829629.798936:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829629.798938:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829629.798942:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787829629.798943:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829629.798945:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787829629.798946:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787829629.798948:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829629.798953:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829629.798959:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c70170c0 x1874670646849024/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787829629.798966:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787829629.798968:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829629.798970:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787829629.798973:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829629.798977:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1787829629.798978:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787829629.798980:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787829629.798983:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787829629.798985:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4430 00000100:00080000:3.0:1787829629.798989:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787829629.798991:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829629.798993:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:3.0:1787829629.798995:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1787829629.798996:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1787829629.798998:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000020:00000001:3.0:1787829629.799000:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:3.0:1787829629.799002:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829629.799004:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:3.0:1787829629.799007:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:3.0:1787829629.799010:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829629.799012:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787829629.799014:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829629.799017:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829629.799018:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:3.0:1787829629.799020:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1787829629.799023:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829629.799025:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829629.799029:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787829629.799031:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829629.799032:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787829629.799034:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787829629.799036:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829629.799040:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829629.799045:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c7017840 x1874670646849152/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787829629.799053:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787829629.799055:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829629.799057:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787829629.799060:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829629.799064:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1787829629.799065:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787829629.799067:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787829629.799070:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787829629.799072:0:468:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4430 00000100:00080000:3.0:1787829629.799076:0:468:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787829629.799078:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829629.799080:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:3.0:1787829629.799083:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1787829629.799084:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1787829629.799086:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000020:00000001:3.0:1787829629.799088:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:3.0:1787829629.799092:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829629.799093:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:3.0:1787829629.799097:0:468:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:3.0:1787829629.799099:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829629.799101:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787829629.799103:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829629.799106:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829629.799108:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:3.0:1787829629.799110:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1787829629.799112:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829629.799114:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829629.799117:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787829629.799119:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829629.799121:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787829629.799122:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787829629.799124:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829629.799128:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829629.799132:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c7014000 x1874670646849280/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787829629.799139:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787829629.799142:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829629.799144:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829629.799236:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787829629.799242:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1787829629.799245:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787829629.799248:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787829629.799250:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829629.799254:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787829629.799257:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829629.799260:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829629.799265:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c7016d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646848896:192.168.203.143@tcp:250:kworker.0 00000100:00000001:1.0:1787829629.799270:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787829629.799274:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787829629.799277:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829629.799280:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787829629.799285:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829629.799300:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787829629.799302:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369547008 : -126234340004608 : ffff8d30c7016d00) 00000100:00000001:1.0:1787829629.799306:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1787829629.799343:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787829629.799354:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787829629.799358:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829629.799361:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829629.799363:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787829629.799369:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829629.799373:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4299104000 00000100:00000001:1.0:1787829629.799382:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 00000100:00000001:1.0:1787829629.799404:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4299104000 00000100:00000001:1.0:1787829629.799410:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 00000100:00000001:1.0:1787829629.799413:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829629.799416:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829629.799419:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787829629.799425:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829629/real 1787829629] req@ffff8d30c7016d00 x1874670646848896/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829654 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787829629.799435:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787829629.799437:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787829629.799441:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787829629.799444:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787829629.799446:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829629.799448:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829629.799450:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787829629.799455:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c7016d00 x1874670646848896/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829654 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787829629.799462:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829629.799466:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787829629.799468:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829629.799470:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829629.799471:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829629.799474:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787829629.799476:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787829629.799479:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829629.799484:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787829629.799486:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787829629.799490:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c7016d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646848896:192.168.203.143@tcp:250:kworker.0 00000100:00000001:1.0:1787829629.799495:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787829629.799496:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787829629.799498:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829629.799500:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787829629.799502:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829629.799504:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829629.799507:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c70170c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646849024:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787829629.799511:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787829629.799513:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787829629.799515:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829629.799517:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787829629.799520:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829629.799526:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787829629.799527:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369547968 : -126234340003648 : ffff8d30c70170c0) 00000100:00000001:1.0:1787829629.799533:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1787829629.799545:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787829629.799550:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787829629.799552:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829629.799554:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829629.799555:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787829629.799558:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829629.799562:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4299102742 00000100:00000001:1.0:1787829629.799565:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4299104000 00000100:00000001:1.0:1787829629.799568:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4299102742 00000100:00000001:1.0:1787829629.799570:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4299104000 00000100:00000001:1.0:1787829629.799572:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829629.799575:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829629.799577:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787829629.799581:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829629/real 1787829629] req@ffff8d30c70170c0 x1874670646849024/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829654 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787829629.799589:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787829629.799590:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787829629.799592:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787829629.799595:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787829629.799596:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829629.799598:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829629.799630:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787829629.799636:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c70170c0 x1874670646849024/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829654 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787829629.799643:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829629.799646:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787829629.799648:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829629.799650:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829629.799652:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829629.799653:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787829629.799655:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787829629.799658:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829629.799662:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787829629.799664:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787829629.799668:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c70170c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646849024:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787829629.799672:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787829629.799674:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787829629.799675:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829629.799677:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787829629.799679:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829629.799681:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829629.799685:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c7017840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646849152:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787829629.799688:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787829629.799690:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787829629.799692:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829629.799693:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787829629.799697:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829629.799703:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787829629.799704:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369549888 : -126234340001728 : ffff8d30c7017840) 00000100:00000001:1.0:1787829629.799708:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1787829629.799735:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787829629.799741:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787829629.799744:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829629.799745:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829629.799747:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787829629.799750:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829629.799755:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4299102743 00000100:00000001:1.0:1787829629.799758:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4299104000 00000100:00000001:1.0:1787829629.799760:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4299102743 00000100:00000001:1.0:1787829629.799762:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4299104000 00000100:00000001:1.0:1787829629.799764:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829629.799766:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829629.799769:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787829629.799774:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829629/real 1787829629] req@ffff8d30c7017840 x1874670646849152/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829654 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787829629.799781:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787829629.799783:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787829629.799785:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787829629.799787:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787829629.799788:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829629.799790:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829629.799791:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787829629.799795:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c7017840 x1874670646849152/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829654 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787829629.799801:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829629.799803:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787829629.799805:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829629.799806:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829629.799808:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829629.799809:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787829629.799811:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787829629.799814:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829629.799817:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787829629.799820:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787829629.799823:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c7017840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646849152:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787829629.799827:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787829629.799829:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787829629.799830:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829629.799833:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787829629.799835:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829629.799837:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829629.799840:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c7014000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646849280:192.168.203.143@tcp:8:kworker.0 00000100:00000001:1.0:1787829629.799844:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787829629.799846:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787829629.799848:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829629.799850:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787829629.799853:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829629.799858:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787829629.799859:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369535488 : -126234340016128 : ffff8d30c7014000) 00000100:00000001:1.0:1787829629.799863:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1787829629.799875:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787829629.799879:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787829629.799881:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829629.799883:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829629.799884:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787829629.799887:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829629.799890:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4299102756 00000100:00000001:1.0:1787829629.799893:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4299104000 00000100:00000001:1.0:1787829629.799895:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4299102756 00000100:00000001:1.0:1787829629.799898:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4299104000 00000100:00000001:1.0:1787829629.799900:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829629.799902:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829629.799905:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787829629.799909:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829629/real 1787829629] req@ffff8d30c7014000 x1874670646849280/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829654 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787829629.799917:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787829629.799918:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787829629.799921:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787829629.799922:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787829629.799924:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829629.799926:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829629.799927:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787829629.799931:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c7014000 x1874670646849280/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829654 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787829629.799938:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829629.799940:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787829629.799943:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829629.799944:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829629.799945:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829629.799947:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787829629.799949:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787829629.799952:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829629.799955:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787829629.799958:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787829629.799961:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c7014000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646849280:192.168.203.143@tcp:8:kworker.0 00000100:00000001:1.0:1787829629.799965:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829629.799967:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829629.799969:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787829629.799971:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787829629.799974:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787829629.799976:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787829629.799978:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787829629.799982:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787829629.799984:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787829629.799987:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787829629.799988:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829629.799990:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829629.799991:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787829629.799993:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787829629.799995:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787829629.799996:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787829629.799997:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787829629.799999:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787829629.800001:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787829629.800002:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787829629.800004:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829629.800005:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829629.800007:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787829629.800008:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787829629.800010:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787829629.800011:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787829629.800013:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787829629.800014:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787829629.800016:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787829629.800017:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787829629.800018:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829629.800020:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829629.800021:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787829629.800022:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787829629.800024:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787829629.800025:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787829629.800026:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787829629.800028:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787829629.800030:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787829629.800031:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787829629.800032:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829629.800035:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829629.800043:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787829629.800045:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829629.800047:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787829629.800049:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829630.694690:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829630.694697:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829630.694759:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829630.694762:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829630.694765:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829631.718728:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829631.718734:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829631.718805:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829631.718809:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829631.718812:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829632.742733:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829632.742739:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829632.742799:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829632.742803:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829632.742807:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829633.766703:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829633.766709:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829633.766772:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829633.766777:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829633.766781:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829634.790697:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829634.790703:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829634.790744:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829634.790748:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829634.790752:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787829634.918663:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829634.918671:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1787829634.918673:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787829634.918676:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787829634.918680:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787829634.918699:0:468:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 4435 00000100:00080000:3.0:1787829634.918709:0:468:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787829634.918712:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829634.918714:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:3.0:1787829634.918718:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1787829634.918720:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1787829634.918722:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:3.0:1787829634.918723:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:3.0:1787829634.918725:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829634.918726:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:3.0:1787829634.918729:0:468:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:3.0:1787829634.918731:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829634.918734:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787829634.918736:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829634.918739:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829634.918740:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:3.0:1787829634.918742:0:468:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829634.918752:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787829634.918753:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829634.918756:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787829634.918758:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787829634.918762:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829634.918776:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829634.918784:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c7017c00 x1874670646849408/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787829634.918792:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787829634.918795:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829634.918807:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787829634.918809:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829634.918812:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1787829634.918813:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787829634.918814:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787829634.918816:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787829634.918818:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4435 00000100:00080000:3.0:1787829634.918820:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787829634.918822:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829634.918823:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:3.0:1787829634.918825:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1787829634.918826:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1787829634.918828:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000020:00000001:3.0:1787829634.918829:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:3.0:1787829634.918831:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829634.918832:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:3.0:1787829634.918834:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:3.0:1787829634.918836:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829634.918838:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1787829634.918842:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:3.0:1787829634.918842:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829634.918846:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829634.918848:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787829634.918848:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:1.0:1787829634.918851:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000008:00100000:3.0:1787829634.918854:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00000001:1.0:1787829634.918855:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787829634.918858:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:3.0:1787829634.918859:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829634.918863:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829634.918864:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787829634.918866:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1787829634.918867:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829634.918870:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829634.918871:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829634.918873:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787829634.918876:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00100000:1.0:1787829634.918878:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c7017c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646849408:192.168.203.143@tcp:250:kworker.0 02000000:00000001:3.0:1787829634.918879:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829634.918885:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829634.918887:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787829634.918891:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000100:00100000:3.0:1787829634.918893:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c7016940 x1874670646849536/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:1.0:1787829634.918894:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829634.918899:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787829634.918905:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829634.918905:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787829634.918908:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829634.918911:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787829634.918913:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829634.918916:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1787829634.918917:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787829634.918919:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787829634.918920:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787829634.918921:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:1.0:1787829634.918923:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369550848 : -126234340000768 : ffff8d30c7017c00) 00000100:00080000:3.0:1787829634.918925:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4435 00000100:00000001:1.0:1787829634.918928:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00080000:3.0:1787829634.918930:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787829634.918933:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829634.918934:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:3.0:1787829634.918935:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1787829634.918936:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1787829634.918937:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000020:00000001:3.0:1787829634.918938:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:3.0:1787829634.918940:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829634.918941:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:3.0:1787829634.918943:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:3.0:1787829634.918945:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829634.918947:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787829634.918949:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829634.918952:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829634.918954:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:3.0:1787829634.918956:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1787829634.918959:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:1.0:1787829634.918963:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:3.0:1787829634.918963:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829634.918972:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787829634.918975:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829634.918977:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787829634.918978:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:1.0:1787829634.918982:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 02000000:00000001:3.0:1787829634.918982:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787829634.918986:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829634.918987:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829634.918992:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787829634.918997:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829634.918997:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829634.919003:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4299109120 00000100:00100000:3.0:1787829634.919008:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c7014000 x1874670646849664/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787829634.919013:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 00000100:00000001:3.0:1787829634.919016:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787829634.919019:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829634.919021:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787829634.919023:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829634.919026:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1787829634.919027:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787829634.919028:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787829634.919030:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:1.0:1787829634.919032:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4299109120 00000100:00080000:3.0:1787829634.919034:0:468:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4435 00000100:00000001:1.0:1787829634.919038:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 00000100:00000001:1.0:1787829634.919042:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00080000:3.0:1787829634.919042:0:468:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1787829634.919047:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829634.919049:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787829634.919050:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00000001:3.0:1787829634.919053:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00100000:1.0:1787829634.919058:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829634/real 1787829634] req@ffff8d30c7017c00 x1874670646849408/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829659 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:3.0:1787829634.919059:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1787829634.919063:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1787829634.919068:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000400:00000001:1.0:1787829634.919071:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000020:00000001:3.0:1787829634.919073:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000400:00000001:1.0:1787829634.919074:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787829634.919078:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829634.919079:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787829634.919081:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829634.919082:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:1.0:1787829634.919085:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829634.919088:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00080000:3.0:1787829634.919089:0:468:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:1.0:1787829634.919090:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829634.919093:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829634.919097:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c7017c00 x1874670646849408/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829659 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1787829634.919098:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787829634.919102:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829634.919107:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829634.919112:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:1.0:1787829634.919114:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000008:00100000:3.0:1787829634.919115:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00000001:1.0:1787829634.919118:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000008:00000001:3.0:1787829634.919119:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829634.919123:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829634.919128:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829634.919130:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829634.919131:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829634.919133:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787829634.919137:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 02000000:00000001:3.0:1787829634.919137:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00080000:1.0:1787829634.919141:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 02000000:00000001:3.0:1787829634.919144:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1787829634.919149:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829634.919149:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787829634.919152:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:1.0:1787829634.919155:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 02000000:00000001:3.0:1787829634.919157:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829634.919164:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c7017c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646849408:192.168.203.143@tcp:250:kworker.0 00000100:00000001:3.0:1787829634.919166:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829634.919172:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829634.919177:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c7017840 x1874670646849792/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787829634.919178:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829634.919181:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787829634.919185:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787829634.919189:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00000001:3.0:1787829634.919190:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000020:00000001:1.0:1787829634.919193:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787829634.919196:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 00000100:00000001:3.0:1787829634.919196:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829634.919201:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 00000100:00000001:3.0:1787829634.919202:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829634.919203:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787829634.919207:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787829634.919209:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829634.919211:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829634.919217:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787829634.919218:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829634.919220:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787829634.919221:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1787829634.919223:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787829634.919224:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787829634.919225:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829634.919227:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787829634.919229:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829634.919231:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829634.919234:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c7016940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646849536:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787829634.919237:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787829634.919239:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787829634.919241:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829634.919242:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787829634.919244:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829634.919249:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787829634.919250:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369546048 : -126234340005568 : ffff8d30c7016940) 00000100:00000001:1.0:1787829634.919254:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1787829634.919263:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787829634.919267:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787829634.919268:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829634.919269:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829634.919270:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787829634.919272:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829634.919275:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4299107862 00000100:00000001:1.0:1787829634.919277:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4299109120 00000100:00000001:1.0:1787829634.919279:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4299107862 00000100:00000001:1.0:1787829634.919280:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4299109120 00000100:00000001:1.0:1787829634.919282:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829634.919283:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829634.919284:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787829634.919287:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829634/real 1787829634] req@ffff8d30c7016940 x1874670646849536/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829659 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787829634.919293:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787829634.919293:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787829634.919295:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787829634.919296:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787829634.919297:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829634.919298:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829634.919299:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787829634.919319:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c7016940 x1874670646849536/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829659 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787829634.919327:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829634.919329:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787829634.919333:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829634.919335:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829634.919337:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829634.919340:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787829634.919342:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787829634.919347:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829634.919352:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787829634.919356:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787829634.919361:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c7016940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646849536:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787829634.919383:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787829634.919385:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787829634.919387:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829634.919389:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787829634.919391:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829634.919394:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829634.919397:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c7014000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646849664:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787829634.919401:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787829634.919403:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787829634.919405:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829634.919407:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787829634.919411:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829634.919418:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787829634.919420:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369535488 : -126234340016128 : ffff8d30c7014000) 00000100:00000001:1.0:1787829634.919424:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1787829634.919452:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787829634.919455:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787829634.919456:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829634.919458:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829634.919459:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787829634.919461:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829634.919463:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4299107862 00000100:00000001:1.0:1787829634.919465:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4299109120 00000100:00000001:1.0:1787829634.919467:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4299107862 00000100:00000001:1.0:1787829634.919470:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4299109120 00000100:00000001:1.0:1787829634.919472:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829634.919474:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829634.919476:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787829634.919480:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829634/real 1787829634] req@ffff8d30c7014000 x1874670646849664/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829659 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787829634.919489:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787829634.919490:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787829634.919493:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787829634.919495:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787829634.919496:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829634.919498:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829634.919499:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787829634.919504:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c7014000 x1874670646849664/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829659 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787829634.919512:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829634.919514:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787829634.919517:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829634.919518:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829634.919519:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829634.919520:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787829634.919521:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787829634.919523:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829634.919525:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787829634.919527:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787829634.919529:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c7014000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646849664:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787829634.919532:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787829634.919533:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787829634.919534:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829634.919535:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787829634.919537:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829634.919538:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829634.919540:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c7017840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646849792:192.168.203.143@tcp:8:kworker.0 00000100:00000001:1.0:1787829634.919542:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787829634.919543:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787829634.919545:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829634.919546:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787829634.919548:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829634.919552:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787829634.919553:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369549888 : -126234340001728 : ffff8d30c7017840) 00000100:00000001:1.0:1787829634.919556:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1787829634.919564:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787829634.919567:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787829634.919569:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829634.919570:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829634.919571:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787829634.919572:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829634.919575:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4299107875 00000100:00000001:1.0:1787829634.919576:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4299109120 00000100:00000001:1.0:1787829634.919578:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4299107875 00000100:00000001:1.0:1787829634.919579:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4299109120 00000100:00000001:1.0:1787829634.919581:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829634.919582:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829634.919583:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787829634.919586:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829634/real 1787829634] req@ffff8d30c7017840 x1874670646849792/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829659 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787829634.919590:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787829634.919591:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787829634.919592:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787829634.919593:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787829634.919594:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829634.919596:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829634.919615:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787829634.919619:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c7017840 x1874670646849792/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829659 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787829634.919626:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829634.919628:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787829634.919630:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829634.919632:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829634.919633:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829634.919641:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787829634.919643:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787829634.919645:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829634.919648:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787829634.919650:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787829634.919653:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c7017840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646849792:192.168.203.143@tcp:8:kworker.0 00000100:00000001:1.0:1787829634.919656:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829634.919658:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829634.919659:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787829634.919661:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787829634.919723:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787829634.919725:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787829634.919727:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787829634.919729:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787829634.919731:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787829634.919733:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787829634.919752:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829634.919754:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829634.919755:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787829634.919757:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787829634.919758:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787829634.919760:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787829634.919761:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787829634.919762:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787829634.919764:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787829634.919765:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787829634.919767:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829634.919768:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829634.919770:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787829634.919771:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787829634.919773:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787829634.919774:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787829634.919775:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787829634.919777:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787829634.919778:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787829634.919780:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787829634.919781:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829634.919783:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829634.919786:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787829634.919788:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829634.919790:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787829634.919792:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829635.814723:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829635.814729:0:815:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1787829635.814731:0:815:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1787829635.814734:0:815:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829635.814738:0:815:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1787829635.814739:0:815:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829635.814741:0:815:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829635.814743:0:815:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829635.814809:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829635.814816:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829635.814822:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829636.838697:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829636.838703:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829636.838808:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829636.838815:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829636.838821:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829637.862682:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829637.862688:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829637.862735:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829637.862738:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829637.862741:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829638.886701:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829638.886707:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829638.886765:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829638.886770:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829638.886773:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829639.910718:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829639.910724:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829639.910772:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829639.910776:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829639.910779:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787829640.038727:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829640.038737:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1787829640.038740:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787829640.038744:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787829640.038748:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787829640.038755:0:40798:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 4440 00000100:00080000:3.0:1787829640.038767:0:40798:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787829640.038770:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829640.038772:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:3.0:1787829640.038776:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1787829640.038779:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1787829640.038782:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:3.0:1787829640.038784:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:3.0:1787829640.038786:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829640.038788:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:3.0:1787829640.038793:0:40798:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:3.0:1787829640.038796:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829640.038799:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787829640.038803:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829640.038806:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829640.038809:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:3.0:1787829640.038812:0:40798:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829640.038824:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787829640.038826:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829640.038829:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787829640.038832:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787829640.038836:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829640.038847:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829640.038857:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c70170c0 x1874670646849920/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787829640.038867:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787829640.038872:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829640.038889:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787829640.038893:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829640.038897:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1787829640.038899:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787829640.038901:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787829640.038904:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787829640.038907:0:40798:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4440 00000100:00080000:3.0:1787829640.038912:0:40798:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787829640.038915:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829640.038917:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:3.0:1787829640.038920:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1787829640.038922:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1787829640.038924:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:1.0:1787829640.038926:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:3.0:1787829640.038928:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:1.0:1787829640.038931:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787829640.038933:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787829640.038934:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787829640.038937:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:1.0:1787829640.038938:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787829640.038941:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829640.038946:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000100:00080000:3.0:1787829640.038947:0:40798:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 02000000:00000001:1.0:1787829640.038949:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829640.038955:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829640.038956:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829640.038961:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c70170c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646849920:192.168.203.143@tcp:250:kworker.0 02000000:00000001:3.0:1787829640.038962:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1787829640.038968:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1787829640.038971:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829640.038972:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787829640.038975:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829640.038977:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829640.038979:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:3.0:1787829640.038982:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 02000000:00000001:1.0:1787829640.038984:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000008:00100000:3.0:1787829640.038988:0:40798:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1787829640.038994:0:40798:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829640.038996:0:40798:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829640.038998:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787829640.039001:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369547968 : -126234340003648 : ffff8d30c70170c0) 02000000:00000001:3.0:1787829640.039002:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:1.0:1787829640.039004:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 02000000:00000001:3.0:1787829640.039006:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829640.039010:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787829640.039012:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787829640.039014:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829640.039021:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829640.039028:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c7016d00 x1874670646850048/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787829640.039038:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000800:00000100:1.0:1787829640.039040:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:3.0:1787829640.039043:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829640.039049:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829640.039051:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00080000:3.0:1787829640.039053:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1787829640.039056:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829640.039059:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787829640.039060:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:1.0:1787829640.039062:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1787829640.039065:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:1.0:1787829640.039068:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787829640.039069:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787829640.039073:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4299114240 00000100:00000001:3.0:1787829640.039075:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787829640.039081:0:40798:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4440 00000100:00000001:1.0:1787829640.039083:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 00000100:00080000:3.0:1787829640.039087:0:40798:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787829640.039091:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787829640.039103:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4299114240 00000100:00000001:1.0:1787829640.039108:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 00000100:00000001:3.0:1787829640.039109:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:1.0:1787829640.039112:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000020:00000001:3.0:1787829640.039115:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:1.0:1787829640.039116:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829640.039120:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000020:00000001:3.0:1787829640.039120:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1787829640.039126:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00100000:1.0:1787829640.039129:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829640/real 1787829640] req@ffff8d30c70170c0 x1874670646849920/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829665 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:3.0:1787829640.039131:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000400:00000001:1.0:1787829640.039143:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000001:3.0:1787829640.039143:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000001:1.0:1787829640.039145:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829640.039148:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:1.0:1787829640.039150:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787829640.039152:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787829640.039155:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:3.0:1787829640.039157:0:40798:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:1.0:1787829640.039158:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829640.039161:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829640.039165:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829640.039168:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c70170c0 x1874670646849920/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829665 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1787829640.039170:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787829640.039176:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829640.039177:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829640.039182:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 02000000:00000001:3.0:1787829640.039183:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829640.039186:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829640.039189:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:1.0:1787829640.039190:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829640.039191:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000008:00100000:3.0:1787829640.039194:0:40798:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00000001:1.0:1787829640.039195:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787829640.039197:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000008:00000001:3.0:1787829640.039202:0:40798:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1787829640.039203:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00000001:3.0:1787829640.039209:0:40798:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1787829640.039210:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787829640.039215:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787829640.039218:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c70170c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646849920:192.168.203.143@tcp:250:kworker.0 02000000:00000001:3.0:1787829640.039219:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787829640.039223:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829640.039225:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829640.039229:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829640.039229:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:1.0:1787829640.039233:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829640.039234:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1787829640.039235:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787829640.039239:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 02000000:00000001:3.0:1787829640.039239:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829640.039242:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787829640.039245:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787829640.039248:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 00000100:00000001:3.0:1787829640.039248:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829640.039251:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787829640.039255:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787829640.039256:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829640.039260:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829640.039261:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c70143c0 x1874670646850176/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787829640.039267:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787829640.039270:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829640.039274:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787829640.039275:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1787829640.039276:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787829640.039277:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787829640.039278:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829640.039280:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787829640.039281:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829640.039283:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829640.039290:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c7016d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646850048:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829640.039290:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787829640.039296:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829640.039297:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787829640.039300:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787829640.039302:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829640.039304:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:3.0:1787829640.039305:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829640.039306:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787829640.039312:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1787829640.039314:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787829640.039317:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369547008 : -126234340004608 : ffff8d30c7016d00) 00000100:00080000:3.0:1787829640.039321:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1787829640.039323:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:3.0:1787829640.039325:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787829640.039329:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787829640.039333:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000800:00000100:1.0:1787829640.039336:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00080000:3.0:1787829640.039341:0:40798:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4440 00000100:00000001:1.0:1787829640.039343:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787829640.039346:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829640.039348:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787829640.039348:0:40798:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1787829640.039351:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1787829640.039355:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787829640.039356:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829640.039360:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:1.0:1787829640.039362:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4299112982 00000020:00000001:3.0:1787829640.039366:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:1.0:1787829640.039367:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4299114240 00000020:00000001:3.0:1787829640.039371:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:1.0:1787829640.039372:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4299112982 00000100:00000001:1.0:1787829640.039376:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4299114240 00000020:00000001:3.0:1787829640.039377:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:1.0:1787829640.039378:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829640.039383:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829640.039385:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000020:00000001:3.0:1787829640.039385:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00100000:1.0:1787829640.039390:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829640/real 1787829640] req@ffff8d30c7016d00 x1874670646850048/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829665 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829640.039392:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829640.039395:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000400:00000001:1.0:1787829640.039403:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787829640.039406:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00080000:3.0:1787829640.039407:0:40798:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:1.0:1787829640.039408:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787829640.039411:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787829640.039414:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829640.039414:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829640.039418:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829640.039421:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:3.0:1787829640.039421:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00100000:1.0:1787829640.039428:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c7016d00 x1874670646850048/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829665 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1787829640.039428:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829640.039436:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829640.039439:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829640.039440:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:1.0:1787829640.039442:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787829640.039445:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000008:00100000:3.0:1787829640.039445:0:40798:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1787829640.039450:0:40798:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829640.039452:0:40798:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829640.039455:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829640.039458:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:3.0:1787829640.039458:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787829640.039463:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829640.039464:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787829640.039467:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829640.039470:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00080000:1.0:1787829640.039471:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 02000000:00000001:3.0:1787829640.039475:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00080000:1.0:1787829640.039476:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787829640.039479:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 02000000:00000001:3.0:1787829640.039480:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829640.039483:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c7016d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646850048:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787829640.039487:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829640.039488:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829640.039489:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829640.039491:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787829640.039495:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787829640.039497:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787829640.039499:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787829640.039500:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787829640.039502:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 00000100:00100000:3.0:1787829640.039502:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c70152c0 x1874670646850304/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:1.0:1787829640.039505:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787829640.039507:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787829640.039509:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829640.039513:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829640.039517:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787829640.039520:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829640.039522:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787829640.039523:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787829640.039526:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787829640.039528:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829640.039529:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787829640.039533:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787829640.039535:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829640.039535:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829640.039542:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787829640.039544:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829640.039545:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829640.039549:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c70143c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646850176:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787829640.039551:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787829640.039552:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787829640.039553:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829640.039555:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787829640.039557:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829640.039561:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787829640.039563:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369536448 : -126234340015168 : ffff8d30c70143c0) 00000100:00000001:1.0:1787829640.039566:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1787829640.039574:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787829640.039577:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787829640.039579:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829640.039580:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829640.039581:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787829640.039583:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829640.039585:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4299112982 00000100:00000001:1.0:1787829640.039587:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4299114240 00000100:00000001:1.0:1787829640.039588:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4299112982 00000100:00000001:1.0:1787829640.039590:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4299114240 00000100:00000001:1.0:1787829640.039591:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829640.039592:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829640.039594:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787829640.039596:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829640/real 1787829640] req@ffff8d30c70143c0 x1874670646850176/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829665 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787829640.039627:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787829640.039628:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787829640.039629:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787829640.039631:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787829640.039632:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829640.039633:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829640.039634:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787829640.039636:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c70143c0 x1874670646850176/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829665 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787829640.039641:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829640.039643:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787829640.039645:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829640.039646:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829640.039660:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829640.039661:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787829640.039663:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787829640.039666:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829640.039668:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787829640.039670:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787829640.039672:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c70143c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646850176:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787829640.039675:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829640.039676:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829640.039677:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787829640.039678:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787829640.039680:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787829640.039681:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787829640.039682:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787829640.039683:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787829640.039684:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787829640.039685:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787829640.039686:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829640.039688:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829640.039690:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787829640.039691:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829640.039692:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787829640.039693:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1787829640.039694:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787829640.039695:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787829640.039696:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829640.039698:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787829640.039699:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829640.039700:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829640.039702:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c70152c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646850304:192.168.203.143@tcp:8:kworker.0 00000100:00000001:1.0:1787829640.039705:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787829640.039707:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787829640.039708:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829640.039710:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787829640.039713:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829640.039719:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787829640.039721:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369540288 : -126234340011328 : ffff8d30c70152c0) 00000100:00000001:1.0:1787829640.039725:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1787829640.039737:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787829640.039742:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787829640.039744:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829640.039746:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829640.039748:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787829640.039751:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829640.039754:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4299112996 00000100:00000001:1.0:1787829640.039757:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4299114240 00000100:00000001:1.0:1787829640.039759:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4299112996 00000100:00000001:1.0:1787829640.039762:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4299114240 00000100:00000001:1.0:1787829640.039764:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829640.039766:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829640.039768:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787829640.039773:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829640/real 1787829640] req@ffff8d30c70152c0 x1874670646850304/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829665 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787829640.039782:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787829640.039783:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787829640.039786:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787829640.039787:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787829640.039789:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829640.039791:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829640.039792:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787829640.039795:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c70152c0 x1874670646850304/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829665 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787829640.039800:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829640.039801:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787829640.039803:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829640.039804:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829640.039804:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829640.039806:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787829640.039807:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787829640.039808:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829640.039811:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787829640.039812:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787829640.039814:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c70152c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646850304:192.168.203.143@tcp:8:kworker.0 00000100:00000001:1.0:1787829640.039817:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829640.039818:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829640.039819:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787829640.039820:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787829640.039821:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787829640.039822:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787829640.039823:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787829640.039824:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787829640.039825:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787829640.039826:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787829640.039827:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829640.039828:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829640.039830:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787829640.039831:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829640.039832:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787829640.039833:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829640.934692:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829640.934699:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829640.934758:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829640.934761:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829640.934764:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829641.958708:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829641.958713:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829641.958780:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829641.958783:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829641.958786:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829642.982679:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829642.982685:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829642.982769:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829642.982776:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829642.982781:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829644.006713:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829644.006718:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829644.006786:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829644.006791:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829644.006795:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829645.030679:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829645.030685:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829645.030736:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829645.030740:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829645.030743:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787829645.158754:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829645.158766:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1787829645.158769:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787829645.158774:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787829645.158779:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787829645.158788:0:40798:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 4445 00000100:00080000:2.0:1787829645.158799:0:40798:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787829645.158803:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829645.158806:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787829645.158811:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787829645.158814:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787829645.158817:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:2.0:1787829645.158820:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:2.0:1787829645.158822:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829645.158823:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:2.0:1787829645.158828:0:40798:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787829645.158830:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829645.158833:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787829645.158836:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787829645.158839:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829645.158841:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:2.0:1787829645.158843:0:40798:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829645.158855:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787829645.158856:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829645.158859:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787829645.158861:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787829645.158866:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829645.158876:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787829645.158884:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2da40 x1874670646850432/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787829645.158894:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787829645.158898:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829645.158914:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787829645.158917:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829645.158920:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1787829645.158922:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787829645.158923:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787829645.158925:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787829645.158927:0:40798:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4445 00000100:00080000:2.0:1787829645.158930:0:40798:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787829645.158932:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829645.158933:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787829645.158935:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787829645.158937:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787829645.158939:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000020:00000001:2.0:1787829645.158941:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:2.0:1787829645.158943:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829645.158945:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:2.0:1787829645.158948:0:40798:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787829645.158950:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829645.158952:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787829645.158957:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787829645.158959:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829645.158960:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787829645.158961:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1787829645.158967:0:40798:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1787829645.158970:0:40798:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829645.158972:0:40798:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829645.158975:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 02000000:00000001:2.0:1787829645.158976:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:1.0:1787829645.158979:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 02000000:00000001:2.0:1787829645.158980:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829645.158984:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787829645.158985:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787829645.158989:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:1.0:1787829645.159001:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829645.159002:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829645.159011:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:2.0:1787829645.159012:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829645.159016:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00100000:2.0:1787829645.159021:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2f480 x1874670646850560/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:1.0:1787829645.159022:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829645.159032:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2da40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646850432:192.168.203.143@tcp:250:kworker.0 00000100:00000001:2.0:1787829645.159037:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787829645.159042:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:2.0:1787829645.159042:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829645.159048:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:2.0:1787829645.159048:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829645.159053:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787829645.159054:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 02000000:00000001:1.0:1787829645.159059:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00080000:2.0:1787829645.159062:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 02000000:00000001:1.0:1787829645.159066:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829645.159067:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787829645.159073:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787829645.159077:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787829645.159082:0:40798:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4445 00000100:00000001:1.0:1787829645.159084:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00080000:2.0:1787829645.159088:0:40798:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1787829645.159089:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417875008 : -126234291676608 : ffff8d30c9e2da40) 00000100:00000001:2.0:1787829645.159096:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787829645.159097:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:2.0:1787829645.159102:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787829645.159107:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787829645.159110:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787829645.159113:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000020:00000001:2.0:1787829645.159116:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:2.0:1787829645.159119:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829645.159121:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:2.0:1787829645.159126:0:40798:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787829645.159128:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829645.159130:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787829645.159133:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787829645.159135:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829645.159137:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1787829645.159139:0:40798:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1787829645.159141:0:40798:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829645.159143:0:40798:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829645.159148:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787829645.159150:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829645.159152:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787829645.159155:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000800:00000100:1.0:1787829645.159157:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 02000000:00000001:2.0:1787829645.159158:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829645.159166:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787829645.159172:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2c000 x1874670646850688/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787829645.159175:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1787829645.159184:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787829645.159185:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787829645.159188:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829645.159191:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829645.159192:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787829645.159200:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1787829645.159201:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00080000:2.0:1787829645.159206:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1787829645.159208:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:1.0:1787829645.159209:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787829645.159210:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787829645.159217:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:1.0:1787829645.159219:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4299119360 00000100:00080000:2.0:1787829645.159220:0:40798:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4445 00000100:00080000:2.0:1787829645.159228:0:40798:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787829645.159231:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829645.159232:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:1.0:1787829645.159233:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 00000020:00000001:2.0:1787829645.159235:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787829645.159238:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787829645.159241:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000020:00000001:2.0:1787829645.159244:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:2.0:1787829645.159247:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829645.159249:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:1.0:1787829645.159250:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4299119360 00000100:00080000:2.0:1787829645.159255:0:40798:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:1.0:1787829645.159258:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 00000100:00000001:2.0:1787829645.159265:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829645.159266:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829645.159273:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829645.159273:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1787829645.159279:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 02000000:00000001:2.0:1787829645.159281:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787829645.159295:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829645.159297:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829645/real 1787829645] req@ffff8d30c9e2da40 x1874670646850432/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829670 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787829645.159301:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1787829645.159307:0:40798:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000400:00000001:1.0:1787829645.159312:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000008:00000001:2.0:1787829645.159312:0:40798:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:1.0:1787829645.159317:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1787829645.159317:0:40798:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829645.159323:0:2390:0:(events.c:78:reply_in_callback()) Process entered 02000000:00000001:2.0:1787829645.159325:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:1.0:1787829645.159328:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 02000000:00000001:2.0:1787829645.159329:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829645.159333:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829645.159334:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787829645.159337:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:1.0:1787829645.159340:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 02000000:00000001:2.0:1787829645.159341:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829645.159343:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829645.159348:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829645.159350:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2da40 x1874670646850432/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829670 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1787829645.159361:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2e940 x1874670646850816/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787829645.159362:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829645.159369:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787829645.159375:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829645.159379:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829645.159382:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829645.159387:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787829645.159387:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787829645.159392:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787829645.159394:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1787829645.159396:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00000001:2.0:1787829645.159402:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1787829645.159404:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787829645.159410:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787829645.159414:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2da40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646850432:192.168.203.143@tcp:250:kworker.0 00000100:00000001:1.0:1787829645.159419:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829645.159421:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829645.159422:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787829645.159425:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787829645.159428:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787829645.159431:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787829645.159433:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787829645.159437:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787829645.159438:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787829645.159442:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787829645.159443:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829645.159445:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829645.159451:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787829645.159453:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829645.159455:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787829645.159457:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1787829645.159458:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787829645.159459:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787829645.159460:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829645.159462:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787829645.159464:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829645.159466:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829645.159468:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646850560:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787829645.159471:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787829645.159473:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787829645.159475:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829645.159476:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787829645.159478:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829645.159483:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787829645.159484:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417881728 : -126234291669888 : ffff8d30c9e2f480) 00000100:00000001:1.0:1787829645.159489:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1787829645.159500:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787829645.159504:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787829645.159506:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829645.159507:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829645.159508:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787829645.159511:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829645.159514:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4299118102 00000100:00000001:1.0:1787829645.159517:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4299119360 00000100:00000001:1.0:1787829645.159519:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4299118102 00000100:00000001:1.0:1787829645.159521:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4299119360 00000100:00000001:1.0:1787829645.159523:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829645.159524:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829645.159525:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787829645.159529:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829645/real 1787829645] req@ffff8d30c9e2f480 x1874670646850560/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829670 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787829645.159535:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787829645.159536:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787829645.159537:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787829645.159539:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787829645.159540:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829645.159541:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829645.159542:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787829645.159545:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2f480 x1874670646850560/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829670 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787829645.159551:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829645.159552:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787829645.159555:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829645.159556:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829645.159557:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829645.159558:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787829645.159560:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787829645.159562:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829645.159564:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787829645.159566:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787829645.159568:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646850560:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787829645.159571:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787829645.159572:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787829645.159573:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829645.159574:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787829645.159576:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829645.159577:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829645.159579:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646850688:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787829645.159581:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787829645.159583:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787829645.159584:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829645.159585:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787829645.159587:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829645.159591:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787829645.159592:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417868288 : -126234291683328 : ffff8d30c9e2c000) 00000100:00000001:1.0:1787829645.159595:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1787829645.159661:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787829645.159666:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787829645.159669:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829645.159670:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829645.159671:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787829645.159674:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829645.159678:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4299118103 00000100:00000001:1.0:1787829645.159680:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4299119360 00000100:00000001:1.0:1787829645.159682:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4299118103 00000100:00000001:1.0:1787829645.159684:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4299119360 00000100:00000001:1.0:1787829645.159686:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829645.159688:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829645.159689:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787829645.159693:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829645/real 1787829645] req@ffff8d30c9e2c000 x1874670646850688/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829670 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787829645.159699:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787829645.159700:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787829645.159702:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787829645.159703:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787829645.159704:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829645.159706:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829645.159707:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787829645.159710:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2c000 x1874670646850688/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829670 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787829645.159716:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829645.159717:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787829645.159719:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829645.159720:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829645.159721:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829645.159723:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787829645.159724:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787829645.159726:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829645.159728:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787829645.159730:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787829645.159732:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646850688:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787829645.159735:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787829645.159736:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787829645.159738:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829645.159739:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787829645.159741:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829645.159742:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829645.159745:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646850816:192.168.203.143@tcp:8:kworker.0 00000100:00000001:1.0:1787829645.159747:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787829645.159749:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787829645.159750:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829645.159751:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787829645.159754:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829645.159758:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787829645.159760:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417878848 : -126234291672768 : ffff8d30c9e2e940) 00000100:00000001:1.0:1787829645.159762:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1787829645.159772:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787829645.159774:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787829645.159776:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829645.159777:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829645.159779:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787829645.159781:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829645.159783:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4299118116 00000100:00000001:1.0:1787829645.159785:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4299119360 00000100:00000001:1.0:1787829645.159787:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4299118116 00000100:00000001:1.0:1787829645.159788:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4299119360 00000100:00000001:1.0:1787829645.159790:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829645.159792:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829645.159793:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787829645.159796:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829645/real 1787829645] req@ffff8d30c9e2e940 x1874670646850816/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829670 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787829645.159801:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787829645.159802:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787829645.159804:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787829645.159805:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787829645.159806:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829645.159807:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829645.159809:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787829645.159812:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2e940 x1874670646850816/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829670 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787829645.159817:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829645.159819:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787829645.159820:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829645.159822:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829645.159823:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829645.159824:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787829645.159825:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787829645.159827:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829645.159829:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787829645.159830:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787829645.159833:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646850816:192.168.203.143@tcp:8:kworker.0 00000100:00000001:1.0:1787829645.159835:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829645.159837:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829645.159838:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787829645.159839:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787829645.159841:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787829645.159842:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787829645.159843:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787829645.159845:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787829645.159846:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787829645.159848:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787829645.159849:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829645.159851:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829645.159852:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787829645.159853:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787829645.159855:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787829645.159856:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787829645.159857:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787829645.159858:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787829645.159859:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787829645.159861:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787829645.159862:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829645.159864:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829645.159865:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787829645.159866:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787829645.159867:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787829645.159868:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787829645.159869:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787829645.159870:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787829645.159871:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787829645.159874:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787829645.159875:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829645.159876:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829645.159879:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787829645.159880:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829645.159881:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787829645.159883:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829646.054736:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829646.054743:0:815:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1787829646.054745:0:815:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1787829646.054747:0:815:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829646.054751:0:815:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1787829646.054754:0:815:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829646.054755:0:815:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829646.054757:0:815:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829646.054822:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829646.054825:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829646.054828:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829647.078689:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829647.078695:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829647.078751:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829647.078755:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829647.078758:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829648.102689:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829648.102694:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829648.102740:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829648.102743:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829648.102746:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829649.126723:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829649.126730:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829649.126839:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829649.126845:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829649.126850:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829650.150705:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829650.150711:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829650.150766:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829650.150769:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829650.150772:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787829650.278723:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829650.278735:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1787829650.278738:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787829650.278742:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787829650.278747:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787829650.278753:0:40798:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 4450 00000100:00080000:0.0:1787829650.278762:0:40798:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787829650.278765:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829650.278767:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787829650.278770:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787829650.278773:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787829650.278775:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:0.0:1787829650.278777:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:0.0:1787829650.278779:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829650.278780:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:0.0:1787829650.278784:0:40798:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787829650.278786:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829650.278788:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787829650.278792:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787829650.278795:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829650.278797:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1787829650.278811:0:40798:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829650.278821:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787829650.278824:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829650.278828:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787829650.278830:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787829650.278835:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829650.278846:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787829650.278856:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d270c0 x1874670646850944/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787829650.278865:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787829650.278868:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829650.278883:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787829650.278886:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829650.278890:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1787829650.278892:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787829650.278893:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787829650.278896:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787829650.278899:0:40798:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4450 00000100:00080000:0.0:1787829650.278904:0:40798:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787829650.278906:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829650.278908:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787829650.278910:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787829650.278911:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787829650.278914:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:3.0:1787829650.278914:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:0.0:1787829650.278919:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:0.0:1787829650.278921:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829650.278921:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787829650.278926:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:3.0:1787829650.278927:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787829650.278932:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00080000:0.0:1787829650.278933:0:40798:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:3.0:1787829650.278936:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829650.278940:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829650.278942:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787829650.278945:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787829650.278946:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829650.278961:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829650.278962:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00100000:3.0:1787829650.278966:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d270c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646850944:192.168.203.143@tcp:250:kworker.0 02000000:00000001:0.0:1787829650.278968:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829650.278973:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:0.0:1787829650.278974:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 02000000:00000001:3.0:1787829650.278978:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000008:00100000:0.0:1787829650.278980:0:40798:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 02000000:00000001:3.0:1787829650.278981:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:0.0:1787829650.278987:0:40798:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829650.278988:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:0.0:1787829650.278989:0:40798:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829650.278995:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787829650.278995:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829650.278998:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829650.279000:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787829650.279001:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787829650.279003:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829650.279007:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787829650.279012:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d24000 x1874670646851072/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787829650.279030:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787829650.279033:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829650.279036:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787829650.279039:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829650.279042:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1787829650.279044:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787829650.279045:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787829650.279048:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787829650.279051:0:40798:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4450 00000100:00080000:0.0:1787829650.279055:0:40798:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787829650.279058:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829650.279059:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787829650.279062:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787829650.279063:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787829650.279065:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000020:00000001:0.0:1787829650.279068:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:0.0:1787829650.279070:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829650.279072:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:0.0:1787829650.279074:0:40798:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787829650.279076:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829650.279078:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787829650.279080:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787829650.279081:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829650.279082:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1787829650.279084:0:40798:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1787829650.279086:0:40798:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829650.279087:0:40798:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829650.279090:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787829650.279091:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829650.279092:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787829650.279093:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787829650.279094:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829650.279097:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787829650.279101:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d24f00 x1874670646851200/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787829650.279116:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787829650.279117:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829650.279119:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787829650.279120:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829650.279131:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1787829650.279132:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787829650.279133:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787829650.279135:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787829650.279137:0:40798:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4450 00000100:00080000:0.0:1787829650.279140:0:40798:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787829650.279141:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829650.279142:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787829650.279143:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787829650.279145:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787829650.279146:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000020:00000001:0.0:1787829650.279148:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:0.0:1787829650.279149:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829650.279150:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:0.0:1787829650.279152:0:40798:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787829650.279153:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829650.279154:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787829650.279155:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787829650.279157:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829650.279158:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1787829650.279159:0:40798:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1787829650.279161:0:40798:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829650.279162:0:40798:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829650.279164:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787829650.279165:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829650.279166:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787829650.279167:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787829650.279169:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829650.279172:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787829650.279176:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d25a40 x1874670646851328/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787829650.279181:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787829650.279182:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829650.279184:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829650.279254:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787829650.279257:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601348800 : -126234108202816 : ffff8d30d4d270c0) 00000100:00000001:3.0:1787829650.279261:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1787829650.279353:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:3.0:1787829650.279363:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787829650.279366:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829650.279368:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829650.279369:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1787829650.279374:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829650.279377:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4299124480 00000100:00000001:3.0:1787829650.279384:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 00000100:00000001:3.0:1787829650.279390:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4299124480 00000100:00000001:3.0:1787829650.279393:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 00000100:00000001:3.0:1787829650.279395:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829650.279397:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829650.279399:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1787829650.279403:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829650/real 1787829650] req@ffff8d30d4d270c0 x1874670646850944/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829675 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1787829650.279421:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787829650.279423:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829650.279426:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829650.279429:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829650.279431:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829650.279434:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829650.279436:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829650.279441:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d270c0 x1874670646850944/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829675 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829650.279449:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829650.279452:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829650.279455:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829650.279457:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829650.279459:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829650.279461:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829650.279464:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829650.279467:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829650.279471:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829650.279474:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829650.279478:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d270c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646850944:192.168.203.143@tcp:250:kworker.0 00000100:00000001:3.0:1787829650.279483:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829650.279484:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829650.279485:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829650.279487:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829650.279490:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829650.279491:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829650.279493:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829650.279496:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829650.279497:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829650.279500:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829650.279501:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829650.279503:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829650.279509:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787829650.279510:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829650.279511:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1787829650.279513:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787829650.279514:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787829650.279515:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787829650.279516:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829650.279518:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787829650.279520:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829650.279521:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829650.279523:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d24000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646851072:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829650.279526:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1787829650.279527:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1787829650.279528:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829650.279529:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1787829650.279531:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829650.279537:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787829650.279538:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601336320 : -126234108215296 : ffff8d30d4d24000) 00000100:00000001:3.0:1787829650.279541:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1787829650.279551:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:3.0:1787829650.279555:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787829650.279557:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829650.279558:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829650.279559:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1787829650.279561:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829650.279564:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4299123222 00000100:00000001:3.0:1787829650.279566:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4299124480 00000100:00000001:3.0:1787829650.279567:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4299123222 00000100:00000001:3.0:1787829650.279569:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4299124480 00000100:00000001:3.0:1787829650.279570:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829650.279571:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829650.279573:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1787829650.279588:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829650/real 1787829650] req@ffff8d30d4d24000 x1874670646851072/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829675 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1787829650.279594:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787829650.279595:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829650.279597:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829650.279598:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829650.279627:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829650.279629:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829650.279630:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829650.279635:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d24000 x1874670646851072/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829675 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829650.279641:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829650.279643:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829650.279645:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829650.279646:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829650.279647:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829650.279649:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829650.279650:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829650.279652:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829650.279655:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829650.279657:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829650.279659:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d24000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646851072:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829650.279662:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787829650.279664:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787829650.279665:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829650.279667:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787829650.279668:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829650.279671:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829650.279674:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d24f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646851200:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829650.279676:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1787829650.279678:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1787829650.279680:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829650.279681:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1787829650.279684:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829650.279689:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787829650.279690:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601340160 : -126234108211456 : ffff8d30d4d24f00) 00000100:00000001:3.0:1787829650.279693:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1787829650.279704:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:3.0:1787829650.279707:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787829650.279709:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829650.279711:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829650.279712:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1787829650.279713:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829650.279727:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4299123223 00000100:00000001:3.0:1787829650.279729:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4299124480 00000100:00000001:3.0:1787829650.279730:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4299123223 00000100:00000001:3.0:1787829650.279732:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4299124480 00000100:00000001:3.0:1787829650.279733:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829650.279734:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829650.279736:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1787829650.279738:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829650/real 1787829650] req@ffff8d30d4d24f00 x1874670646851200/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829675 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1787829650.279743:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787829650.279744:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829650.279745:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829650.279746:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829650.279747:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829650.279748:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829650.279749:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829650.279752:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d24f00 x1874670646851200/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829675 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829650.279756:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829650.279757:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829650.279758:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829650.279759:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829650.279760:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829650.279761:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829650.279762:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829650.279764:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829650.279765:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829650.279767:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829650.279769:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d24f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646851200:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829650.279772:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787829650.279773:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787829650.279774:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829650.279775:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787829650.279776:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829650.279777:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829650.279779:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d25a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646851328:192.168.203.143@tcp:8:kworker.0 00000100:00000001:3.0:1787829650.279780:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1787829650.279782:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1787829650.279783:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829650.279784:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1787829650.279786:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829650.279789:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787829650.279790:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601343040 : -126234108208576 : ffff8d30d4d25a40) 00000100:00000001:3.0:1787829650.279792:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1787829650.279798:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:3.0:1787829650.279801:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787829650.279802:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829650.279803:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829650.279804:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1787829650.279806:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829650.279808:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4299123236 00000100:00000001:3.0:1787829650.279809:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4299124480 00000100:00000001:3.0:1787829650.279810:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4299123236 00000100:00000001:3.0:1787829650.279812:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4299124480 00000100:00000001:3.0:1787829650.279813:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829650.279814:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829650.279815:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1787829650.279818:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829650/real 1787829650] req@ffff8d30d4d25a40 x1874670646851328/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829675 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1787829650.279827:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787829650.279828:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829650.279829:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829650.279830:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829650.279831:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829650.279832:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829650.279833:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829650.279835:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d25a40 x1874670646851328/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829675 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829650.279839:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829650.279840:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829650.279842:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829650.279843:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829650.279843:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829650.279844:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829650.279845:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829650.279847:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829650.279849:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829650.279850:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829650.279852:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d25a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646851328:192.168.203.143@tcp:8:kworker.0 00000100:00000001:3.0:1787829650.279854:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829650.279855:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829650.279856:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829650.279857:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829650.279859:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829650.279860:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829650.279861:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829650.279862:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829650.279863:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829650.279865:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829650.279866:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829650.279867:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829650.279868:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829650.279869:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829650.279870:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829650.279871:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829650.279872:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829650.279873:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829650.279873:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829650.279875:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829650.279875:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829650.279876:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829650.279877:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829650.279878:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829650.279879:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829650.279879:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829650.279880:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829650.279881:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829650.279881:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829650.279883:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829650.279884:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829650.279885:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829650.279888:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787829650.279890:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829650.279892:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1787829650.279893:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829651.174690:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829651.174695:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829651.174748:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829651.174751:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829651.174754:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829652.198735:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829652.198743:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829652.198819:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829652.198825:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829652.198828:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829653.222671:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829653.222678:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829653.222731:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829653.222736:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829653.222740:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829654.246701:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829654.246705:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829654.246804:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829654.246812:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829654.246817:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829655.270672:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829655.270679:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829655.270737:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829655.270740:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829655.270743:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787829655.398713:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829655.398732:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1787829655.398735:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787829655.398740:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787829655.398744:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787829655.398751:0:40798:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 4455 00000100:00080000:2.0:1787829655.398761:0:40798:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787829655.398764:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829655.398766:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787829655.398771:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787829655.398773:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787829655.398776:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:2.0:1787829655.398779:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:2.0:1787829655.398781:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829655.398783:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:2.0:1787829655.398787:0:40798:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787829655.398790:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829655.398794:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787829655.398798:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787829655.398801:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829655.398803:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:2.0:1787829655.398806:0:40798:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829655.398817:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787829655.398819:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829655.398822:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787829655.398825:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787829655.398830:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829655.398841:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787829655.398851:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2ed00 x1874670646851456/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787829655.398862:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787829655.398866:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829655.398881:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787829655.398884:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829655.398888:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1787829655.398889:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787829655.398891:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787829655.398894:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787829655.398897:0:40798:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4455 00000100:00080000:2.0:1787829655.398903:0:40798:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787829655.398905:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829655.398907:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:3.0:1787829655.398910:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:2.0:1787829655.398911:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787829655.398915:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:3.0:1787829655.398916:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000020:00000001:2.0:1787829655.398918:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:3.0:1787829655.398920:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787829655.398923:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000020:00000001:2.0:1787829655.398924:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:3.0:1787829655.398926:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829655.398931:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 02000000:00000001:3.0:1787829655.398933:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:2.0:1787829655.398935:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 02000000:00000001:3.0:1787829655.398936:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829655.398942:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787829655.398945:0:40798:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00100000:3.0:1787829655.398948:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646851456:192.168.203.143@tcp:250:kworker.0 00000100:00000001:2.0:1787829655.398950:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829655.398955:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:3.0:1787829655.398959:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1787829655.398963:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1787829655.398964:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829655.398966:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829655.398969:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829655.398970:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:2.0:1787829655.398974:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 02000000:00000001:3.0:1787829655.398976:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000008:00100000:2.0:1787829655.398980:0:40798:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1787829655.398986:0:40798:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829655.398990:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787829655.398993:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417879808 : -126234291671808 : ffff8d30c9e2ed00) 00000100:00000001:3.0:1787829655.398996:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:2.0:1787829655.398998:0:40798:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829655.399005:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787829655.399007:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829655.399009:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787829655.399011:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787829655.399013:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829655.399018:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787829655.399025:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2f0c0 x1874670646851584/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1787829655.399034:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:2.0:1787829655.399039:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787829655.399042:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829655.399045:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787829655.399049:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829655.399053:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1787829655.399055:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:3.0:1787829655.399055:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00080000:2.0:1787829655.399058:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787829655.399061:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787829655.399063:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:3.0:1787829655.399064:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829655.399068:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00080000:2.0:1787829655.399070:0:40798:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4455 00000100:00000001:3.0:1787829655.399073:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787829655.399078:0:40798:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787829655.399078:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4299129600 00000100:00000001:2.0:1787829655.399085:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829655.399087:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:3.0:1787829655.399088:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 00000020:00000001:2.0:1787829655.399094:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:3.0:1787829655.399096:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4299129600 00000020:00000001:2.0:1787829655.399098:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:3.0:1787829655.399101:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 00000020:00000001:2.0:1787829655.399102:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:3.0:1787829655.399105:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000020:00000001:2.0:1787829655.399108:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:3.0:1787829655.399109:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829655.399114:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829655.399118:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:3.0:1787829655.399119:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1787829655.399125:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829655/real 1787829655] req@ffff8d30c9e2ed00 x1874670646851456/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829680 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829655.399128:0:40798:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787829655.399134:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829655.399140:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000400:00000001:3.0:1787829655.399142:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 02000000:00000001:2.0:1787829655.399145:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000001:3.0:1787829655.399145:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829655.399150:0:2390:0:(events.c:78:reply_in_callback()) Process entered 02000000:00000001:2.0:1787829655.399151:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829655.399153:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787829655.399155:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:3.0:1787829655.399157:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000008:00100000:2.0:1787829655.399159:0:40798:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00000001:3.0:1787829655.399163:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000008:00000001:2.0:1787829655.399164:0:40798:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829655.399167:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829655.399176:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2ed00 x1874670646851456/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829680 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787829655.399178:0:40798:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829655.399186:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787829655.399188:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829655.399190:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829655.399193:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787829655.399196:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:3.0:1787829655.399197:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 02000000:00000001:2.0:1787829655.399199:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829655.399203:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829655.399207:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829655.399207:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829655.399209:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829655.399214:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829655.399217:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00100000:2.0:1787829655.399219:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2c780 x1874670646851712/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1787829655.399223:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829655.399233:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:2.0:1787829655.399236:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787829655.399238:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000001:2.0:1787829655.399241:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829655.399246:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646851456:192.168.203.143@tcp:250:kworker.0 00000100:00000001:2.0:1787829655.399248:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787829655.399254:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1787829655.399256:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:2.0:1787829655.399261:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1787829655.399262:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787829655.399265:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:3.0:1787829655.399266:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 00000100:00080000:2.0:1787829655.399269:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 02000000:00000001:3.0:1787829655.399271:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 00000100:00000001:2.0:1787829655.399276:0:40798:0:(import.c:485:import_select_connection()) Process entered 02000000:00000001:3.0:1787829655.399276:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829655.399281:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000100:00080000:2.0:1787829655.399282:0:40798:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4455 00000020:00000001:3.0:1787829655.399285:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 00000100:00080000:2.0:1787829655.399289:0:40798:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 02000000:00000001:3.0:1787829655.399292:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 00000100:00000001:2.0:1787829655.399293:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 02000000:00000001:3.0:1787829655.399296:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1787829655.399297:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787829655.399302:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787829655.399304:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:3.0:1787829655.399304:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000020:00000001:2.0:1787829655.399307:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:3.0:1787829655.399308:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:2.0:1787829655.399314:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:3.0:1787829655.399315:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829655.399318:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829655.399322:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:3.0:1787829655.399325:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00080000:2.0:1787829655.399327:0:40798:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:3.0:1787829655.399329:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829655.399335:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829655.399336:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1787829655.399340:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 02000000:00000001:2.0:1787829655.399341:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:3.0:1787829655.399343:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 02000000:00000001:2.0:1787829655.399346:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829655.399347:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787829655.399350:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829655.399352:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829655.399354:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:2.0:1787829655.399358:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 02000000:00000001:3.0:1787829655.399358:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000008:00100000:2.0:1787829655.399363:0:40798:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 02000000:00000001:3.0:1787829655.399364:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:2.0:1787829655.399371:0:40798:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829655.399373:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646851584:192.168.203.143@tcp:38:kworker.0 00000100:00000001:2.0:1787829655.399375:0:40798:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829655.399381:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787829655.399384:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787829655.399385:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1787829655.399387:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829655.399389:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829655.399391:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787829655.399394:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1787829655.399395:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787829655.399399:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829655.399400:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829655.399407:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829655.399408:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787829655.399411:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417880768 : -126234291670848 : ffff8d30c9e2f0c0) 00000100:00000001:3.0:1787829655.399416:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00100000:2.0:1787829655.399417:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2fc00 x1874670646851840/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787829655.399428:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000800:00000100:3.0:1787829655.399430:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:2.0:1787829655.399432:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829655.399435:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1787829655.399439:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829655.399440:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829655.399442:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829655.399445:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1787829655.399448:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829655.399451:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4299128342 00000100:00000001:3.0:1787829655.399454:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4299129600 00000100:00000001:3.0:1787829655.399456:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4299128342 00000100:00000001:3.0:1787829655.399458:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4299129600 00000100:00000001:3.0:1787829655.399460:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829655.399461:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829655.399463:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1787829655.399466:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829655/real 1787829655] req@ffff8d30c9e2f0c0 x1874670646851584/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829680 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1787829655.399472:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787829655.399473:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829655.399474:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829655.399476:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829655.399477:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829655.399478:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829655.399479:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829655.399481:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2f0c0 x1874670646851584/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829680 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829655.399486:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829655.399487:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829655.399489:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829655.399490:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829655.399491:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829655.399492:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829655.399494:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829655.399495:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829655.399497:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829655.399498:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829655.399500:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646851584:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829655.399503:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787829655.399504:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787829655.399504:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829655.399506:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787829655.399507:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829655.399508:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829655.399510:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646851712:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829655.399512:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1787829655.399513:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1787829655.399514:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829655.399515:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1787829655.399517:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829655.399520:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787829655.399521:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417870208 : -126234291681408 : ffff8d30c9e2c780) 00000100:00000001:3.0:1787829655.399523:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1787829655.399530:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:3.0:1787829655.399532:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787829655.399533:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829655.399534:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829655.399536:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1787829655.399537:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829655.399553:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4299128342 00000100:00000001:3.0:1787829655.399555:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4299129600 00000100:00000001:3.0:1787829655.399557:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4299128342 00000100:00000001:3.0:1787829655.399559:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4299129600 00000100:00000001:3.0:1787829655.399561:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829655.399562:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829655.399563:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1787829655.399566:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829655/real 1787829655] req@ffff8d30c9e2c780 x1874670646851712/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829680 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1787829655.399571:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787829655.399572:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829655.399574:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829655.399575:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829655.399576:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829655.399577:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829655.399578:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829655.399580:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2c780 x1874670646851712/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829680 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829655.399585:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829655.399586:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829655.399587:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829655.399588:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829655.399589:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829655.399590:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829655.399591:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829655.399593:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829655.399594:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829655.399596:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829655.399597:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646851712:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829655.399627:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829655.399628:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829655.399629:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829655.399631:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829655.399632:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829655.399633:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829655.399634:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829655.399636:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829655.399637:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829655.399638:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829655.399639:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829655.399640:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829655.399641:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829655.399642:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829655.399643:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829655.399644:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829655.399645:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829655.399646:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829655.399646:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829655.399648:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829655.399648:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829655.399649:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829655.399652:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787829655.399652:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829655.399654:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1787829655.399655:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787829655.399656:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787829655.399657:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787829655.399658:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829655.399659:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787829655.399660:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829655.399662:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829655.399664:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646851840:192.168.203.143@tcp:8:kworker.0 00000100:00000001:3.0:1787829655.399666:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1787829655.399667:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1787829655.399668:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829655.399676:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1787829655.399678:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829655.399682:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787829655.399683:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417883648 : -126234291667968 : ffff8d30c9e2fc00) 00000100:00000001:3.0:1787829655.399686:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1787829655.399694:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:3.0:1787829655.399698:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787829655.399699:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829655.399700:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829655.399701:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1787829655.399703:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829655.399705:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4299128356 00000100:00000001:3.0:1787829655.399707:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4299129600 00000100:00000001:3.0:1787829655.399708:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4299128356 00000100:00000001:3.0:1787829655.399710:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4299129600 00000100:00000001:3.0:1787829655.399711:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829655.399712:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829655.399714:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1787829655.399716:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829655/real 1787829655] req@ffff8d30c9e2fc00 x1874670646851840/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829680 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1787829655.399721:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787829655.399722:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829655.399723:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829655.399724:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829655.399725:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829655.399726:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829655.399727:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829655.399729:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2fc00 x1874670646851840/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829680 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829655.399734:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829655.399735:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829655.399736:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829655.399737:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829655.399738:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829655.399739:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829655.399740:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829655.399742:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829655.399744:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829655.399746:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829655.399748:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646851840:192.168.203.143@tcp:8:kworker.0 00000100:00000001:3.0:1787829655.399750:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829655.399751:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829655.399752:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829655.399753:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829655.399754:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829655.399755:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829655.399755:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829655.399756:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829655.399757:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829655.399758:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829655.399759:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829655.399760:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829655.399762:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787829655.399763:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829655.399764:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1787829655.399765:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829656.294711:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829656.294719:0:815:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1787829656.294721:0:815:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1787829656.294726:0:815:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829656.294732:0:815:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1787829656.294735:0:815:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829656.294739:0:815:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829656.294741:0:815:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829656.294789:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829656.294793:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829656.294797:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829657.318666:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829657.318670:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829657.318719:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829657.318723:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829657.318726:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829658.342694:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829658.342700:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829658.342823:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829658.342830:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829658.342835:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829659.366707:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829659.366712:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829659.366761:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829659.366764:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829659.366766:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829660.390691:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829660.390697:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829660.390771:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829660.390774:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829660.390777:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787829660.518724:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829660.518732:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1787829660.518734:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787829660.518737:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787829660.518741:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787829660.518747:0:40798:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 4461 00000100:00080000:0.0:1787829660.518755:0:40798:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787829660.518758:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829660.518759:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787829660.518762:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787829660.518764:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787829660.518767:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:0.0:1787829660.518769:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:0.0:1787829660.518770:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829660.518771:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:0.0:1787829660.518774:0:40798:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787829660.518776:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829660.518778:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787829660.518780:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787829660.518783:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829660.518784:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1787829660.518786:0:40798:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829660.518797:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787829660.518798:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829660.518800:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787829660.518802:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787829660.518805:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829660.518815:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787829660.518822:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d252c0 x1874670646851968/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787829660.518830:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787829660.518833:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829660.518847:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787829660.518849:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829660.518852:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1787829660.518853:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787829660.518854:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787829660.518856:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787829660.518857:0:40798:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4461 00000100:00080000:0.0:1787829660.518860:0:40798:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787829660.518861:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829660.518862:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787829660.518864:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787829660.518865:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787829660.518866:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000020:00000001:0.0:1787829660.518868:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:0.0:1787829660.518869:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829660.518871:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:0.0:1787829660.518873:0:40798:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787829660.518875:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829660.518876:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787829660.518880:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787829660.518882:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829660.518883:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1787829660.518887:0:40798:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1787829660.518890:0:40798:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829660.518892:0:40798:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829660.518895:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787829660.518896:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829660.518898:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787829660.518900:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787829660.518902:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829660.518907:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829660.518910:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00100000:0.0:1787829660.518914:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d25680 x1874670646852096/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787829660.518918:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787829660.518923:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787829660.518927:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787829660.518931:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829660.518932:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787829660.518941:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829660.518941:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787829660.518946:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829660.518950:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829660.518957:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d252c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646851968:192.168.203.143@tcp:250:kworker.0 00000100:00000001:0.0:1787829660.518960:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787829660.518965:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829660.518967:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1787829660.518969:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787829660.518971:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787829660.518971:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:0.0:1787829660.518976:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787829660.518982:0:40798:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4461 02000000:00000001:3.0:1787829660.518983:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1787829660.518987:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787829660.518990:0:40798:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 02000000:00000001:3.0:1787829660.518991:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:0.0:1787829660.518997:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829660.518998:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 02000000:00000001:3.0:1787829660.519001:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787829660.519003:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787829660.519007:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787829660.519009:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000020:00000001:0.0:1787829660.519013:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:0.0:1787829660.519017:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829660.519018:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:3.0:1787829660.519021:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787829660.519025:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601341120 : -126234108210496 : ffff8d30d4d252c0) 00000100:00080000:0.0:1787829660.519026:0:40798:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:3.0:1787829660.519031:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:0.0:1787829660.519033:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829660.519037:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787829660.519041:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787829660.519043:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829660.519045:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1787829660.519048:0:40798:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1787829660.519050:0:40798:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829660.519051:0:40798:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829660.519055:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787829660.519056:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829660.519058:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787829660.519059:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787829660.519060:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829660.519064:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787829660.519070:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d27c00 x1874670646852224/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787829660.519076:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787829660.519078:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829660.519080:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787829660.519082:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829660.519084:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1787829660.519085:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787829660.519087:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:3.0:1787829660.519093:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:0.0:1787829660.519094:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787829660.519101:0:40798:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4461 00000100:00080000:0.0:1787829660.519106:0:40798:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787829660.519107:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829660.519108:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787829660.519110:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787829660.519111:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787829660.519112:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000020:00000001:0.0:1787829660.519113:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:0.0:1787829660.519116:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829660.519117:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:0.0:1787829660.519120:0:40798:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787829660.519122:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829660.519123:0:2390:0:(events.c:34:request_out_callback()) Process entered 02000000:00000001:0.0:1787829660.519124:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787829660.519128:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829660.519130:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 02000000:00000001:0.0:1787829660.519131:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829660.519138:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829660.519139:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:3.0:1787829660.519147:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000008:00100000:0.0:1787829660.519148:0:40798:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1787829660.519153:0:40798:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829660.519155:0:40798:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829660.519158:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829660.519159:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787829660.519166:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829660.519169:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787829660.519170:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:3.0:1787829660.519171:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4299134720 02000000:00000001:0.0:1787829660.519173:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829660.519184:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787829660.519188:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d26940 x1874670646852352/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787829660.519194:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1787829660.519195:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787829660.519200:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829660.519203:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829660.519209:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4299134720 00000100:00000001:3.0:1787829660.519216:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 00000100:00000001:3.0:1787829660.519224:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829660.519228:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829660.519231:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1787829660.519240:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829660/real 1787829660] req@ffff8d30d4d252c0 x1874670646851968/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829685 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1787829660.519250:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787829660.519252:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829660.519255:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829660.519258:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829660.519259:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829660.519261:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829660.519263:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829660.519267:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d252c0 x1874670646851968/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829685 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829660.519273:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829660.519276:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829660.519279:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829660.519280:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829660.519281:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829660.519284:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829660.519286:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829660.519289:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829660.519293:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829660.519294:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829660.519298:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d252c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646851968:192.168.203.143@tcp:250:kworker.0 00000100:00000001:3.0:1787829660.519304:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829660.519306:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829660.519308:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829660.519310:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829660.519314:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829660.519318:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829660.519320:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829660.519325:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829660.519327:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829660.519332:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829660.519333:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829660.519336:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829660.519345:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787829660.519347:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829660.519350:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1787829660.519352:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787829660.519353:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787829660.519355:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787829660.519356:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829660.519358:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787829660.519360:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829660.519361:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829660.519364:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d25680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646852096:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829660.519367:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1787829660.519369:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1787829660.519370:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829660.519371:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1787829660.519374:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829660.519379:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787829660.519380:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601342080 : -126234108209536 : ffff8d30d4d25680) 00000100:00000001:3.0:1787829660.519385:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1787829660.519396:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:3.0:1787829660.519400:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787829660.519402:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829660.519403:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829660.519404:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1787829660.519406:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829660.519410:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4299133462 00000100:00000001:3.0:1787829660.519412:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4299134720 00000100:00000001:3.0:1787829660.519414:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4299133462 00000100:00000001:3.0:1787829660.519416:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4299134720 00000100:00000001:3.0:1787829660.519417:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829660.519419:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829660.519420:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1787829660.519422:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829660/real 1787829660] req@ffff8d30d4d25680 x1874670646852096/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829685 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1787829660.519428:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787829660.519429:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829660.519430:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829660.519431:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829660.519432:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829660.519434:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829660.519435:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829660.519437:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d25680 x1874670646852096/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829685 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829660.519441:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829660.519443:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829660.519444:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829660.519445:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829660.519446:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829660.519448:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829660.519449:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829660.519451:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829660.519454:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829660.519455:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829660.519457:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d25680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646852096:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829660.519459:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787829660.519460:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787829660.519461:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829660.519463:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787829660.519464:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829660.519465:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829660.519467:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d27c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646852224:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829660.519470:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1787829660.519471:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1787829660.519472:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829660.519473:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1787829660.519475:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829660.519478:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787829660.519479:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601351680 : -126234108199936 : ffff8d30d4d27c00) 00000100:00000001:3.0:1787829660.519482:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1787829660.519488:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:3.0:1787829660.519491:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787829660.519492:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829660.519493:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829660.519494:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1787829660.519496:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829660.519498:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4299133462 00000100:00000001:3.0:1787829660.519500:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4299134720 00000100:00000001:3.0:1787829660.519502:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4299133462 00000100:00000001:3.0:1787829660.519520:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4299134720 00000100:00000001:3.0:1787829660.519523:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829660.519524:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829660.519526:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1787829660.519529:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829660/real 1787829660] req@ffff8d30d4d27c00 x1874670646852224/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829685 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1787829660.519535:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787829660.519536:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829660.519538:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829660.519539:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829660.519540:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829660.519541:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829660.519542:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829660.519545:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d27c00 x1874670646852224/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829685 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829660.519549:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829660.519551:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829660.519552:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829660.519553:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829660.519554:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829660.519555:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829660.519556:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829660.519557:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829660.519559:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829660.519560:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829660.519562:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d27c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646852224:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829660.519565:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787829660.519566:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787829660.519567:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829660.519569:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787829660.519570:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829660.519572:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829660.519574:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d26940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646852352:192.168.203.143@tcp:8:kworker.0 00000100:00000001:3.0:1787829660.519577:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1787829660.519578:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1787829660.519579:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829660.519581:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1787829660.519583:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829660.519587:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787829660.519588:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601346880 : -126234108204736 : ffff8d30d4d26940) 00000100:00000001:3.0:1787829660.519590:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1787829660.519598:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:3.0:1787829660.519631:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787829660.519633:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829660.519634:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829660.519635:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1787829660.519637:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829660.519639:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4299133476 00000100:00000001:3.0:1787829660.519641:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4299134720 00000100:00000001:3.0:1787829660.519643:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4299133476 00000100:00000001:3.0:1787829660.519645:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4299134720 00000100:00000001:3.0:1787829660.519646:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829660.519647:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829660.519648:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1787829660.519651:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829660/real 1787829660] req@ffff8d30d4d26940 x1874670646852352/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829685 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1787829660.519656:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787829660.519657:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829660.519658:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829660.519659:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829660.519660:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829660.519661:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829660.519663:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829660.519665:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d26940 x1874670646852352/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829685 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829660.519675:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829660.519676:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829660.519678:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829660.519679:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829660.519680:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829660.519681:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829660.519682:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829660.519683:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829660.519685:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829660.519687:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829660.519688:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d26940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646852352:192.168.203.143@tcp:8:kworker.0 00000100:00000001:3.0:1787829660.519691:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829660.519692:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829660.519693:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829660.519694:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829660.519695:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829660.519696:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829660.519697:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829660.519699:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829660.519700:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829660.519701:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829660.519702:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829660.519703:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829660.519704:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829660.519705:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829660.519706:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829660.519707:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829660.519708:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829660.519709:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829660.519709:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829660.519712:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829660.519713:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829660.519714:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829660.519714:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829660.519716:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829660.519717:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829660.519718:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829660.519719:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829660.519720:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829660.519721:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829660.519723:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829660.519724:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829660.519725:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829660.519727:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787829660.519728:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829660.519729:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1787829660.519730:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829661.414728:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829661.414735:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829661.414800:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829661.414809:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829661.414813:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829662.438652:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829662.438658:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829662.438699:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829662.438702:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829662.438705:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829663.462705:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829663.462712:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829663.462821:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829663.462826:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829663.462830:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829664.486695:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829664.486701:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829664.486786:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829664.486791:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829664.486795:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829665.510705:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829665.510713:0:815:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1787829665.510716:0:815:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1787829665.510718:0:815:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829665.510722:0:815:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1787829665.510724:0:815:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829665.510725:0:815:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829665.510727:0:815:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829665.510791:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829665.510795:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829665.510799:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787829665.638704:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829665.638712:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1787829665.638713:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787829665.638716:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787829665.638720:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787829665.638726:0:40798:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 4466 00000100:00080000:2.0:1787829665.638734:0:40798:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787829665.638737:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829665.638739:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787829665.638742:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787829665.638744:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787829665.638746:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:2.0:1787829665.638748:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:2.0:1787829665.638749:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829665.638750:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:2.0:1787829665.638753:0:40798:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787829665.638755:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829665.638757:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787829665.638759:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787829665.638761:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829665.638763:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:2.0:1787829665.638765:0:40798:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829665.638775:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787829665.638776:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829665.638779:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787829665.638781:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787829665.638785:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829665.638795:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787829665.638802:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2cb40 x1874670646852480/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787829665.638809:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787829665.638813:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829665.638825:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787829665.638827:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829665.638830:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1787829665.638831:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787829665.638832:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787829665.638834:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787829665.638835:0:40798:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4466 00000100:00080000:2.0:1787829665.638838:0:40798:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787829665.638839:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829665.638841:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787829665.638842:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787829665.638844:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787829665.638845:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000020:00000001:2.0:1787829665.638847:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:2.0:1787829665.638849:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829665.638850:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:2.0:1787829665.638852:0:40798:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787829665.638854:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829665.638855:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:3.0:1787829665.638857:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:2.0:1787829665.638859:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787829665.638863:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829665.638863:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1787829665.638866:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:3.0:1787829665.638868:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787829665.638871:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787829665.638872:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829665.638876:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000008:00100000:2.0:1787829665.638877:0:40798:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 02000000:00000001:3.0:1787829665.638879:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000008:00000001:2.0:1787829665.638881:0:40798:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829665.638883:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829665.638884:0:40798:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829665.638889:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00100000:3.0:1787829665.638890:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646852480:192.168.203.143@tcp:250:kworker.0 02000000:00000001:2.0:1787829665.638891:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829665.638895:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:3.0:1787829665.638898:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787829665.638899:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787829665.638903:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829665.638904:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:2.0:1787829665.638908:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829665.638908:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787829665.638914:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2c3c0 x1874670646852608/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:3.0:1787829665.638920:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:2.0:1787829665.638924:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787829665.638926:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829665.638927:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829665.638929:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787829665.638932:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829665.638935:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1787829665.638937:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787829665.638938:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787829665.638940:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787829665.638942:0:40798:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4466 00000100:00080000:2.0:1787829665.638945:0:40798:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787829665.638945:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787829665.638949:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829665.638950:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417871168 : -126234291680448 : ffff8d30c9e2cb40) 00000100:00000001:2.0:1787829665.638951:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787829665.638955:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787829665.638957:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787829665.638959:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000020:00000001:2.0:1787829665.638961:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:3.0:1787829665.638961:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:2.0:1787829665.638965:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829665.638966:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:2.0:1787829665.638968:0:40798:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787829665.638970:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829665.638971:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787829665.638972:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787829665.638974:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829665.638975:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1787829665.638976:0:40798:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1787829665.638978:0:40798:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829665.638979:0:40798:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829665.638983:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787829665.638983:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829665.638984:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787829665.638985:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787829665.638987:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829665.638991:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787829665.638994:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2d2c0 x1874670646852736/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787829665.638999:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787829665.639000:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829665.639002:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787829665.639006:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:3.0:1787829665.639007:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00080000:2.0:1787829665.639009:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1787829665.639012:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787829665.639013:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787829665.639015:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787829665.639022:0:40798:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4466 00000100:00000001:3.0:1787829665.639022:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00080000:2.0:1787829665.639029:0:40798:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787829665.639029:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787829665.639033:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829665.639033:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829665.639037:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:3.0:1787829665.639038:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000020:00000001:2.0:1787829665.639041:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787829665.639043:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787829665.639044:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:3.0:1787829665.639045:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1787829665.639046:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:2.0:1787829665.639051:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829665.639052:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:3.0:1787829665.639053:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4299139840 00000100:00080000:2.0:1787829665.639056:0:40798:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787829665.639059:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829665.639060:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787829665.639062:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829665.639063:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 02000000:00000001:2.0:1787829665.639064:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829665.639067:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1787829665.639069:0:40798:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000008:00000001:2.0:1787829665.639070:0:40798:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829665.639071:0:40798:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829665.639075:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787829665.639075:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829665.639077:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:3.0:1787829665.639077:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4299139840 02000000:00000001:2.0:1787829665.639078:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787829665.639081:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829665.639085:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 00000100:00000001:2.0:1787829665.639087:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829665.639091:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00100000:2.0:1787829665.639093:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2de00 x1874670646852864/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787829665.639094:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829665.639100:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00000001:2.0:1787829665.639103:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787829665.639106:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829665.639109:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829665.639109:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829665/real 1787829665] req@ffff8d30c9e2cb40 x1874670646852480/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829690 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1787829665.639121:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787829665.639123:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829665.639125:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829665.639129:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829665.639130:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829665.639132:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829665.639134:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829665.639137:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2cb40 x1874670646852480/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829690 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829665.639143:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829665.639145:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829665.639147:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829665.639149:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829665.639149:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829665.639151:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829665.639153:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829665.639154:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829665.639157:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829665.639158:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829665.639161:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646852480:192.168.203.143@tcp:250:kworker.0 00000100:00000001:3.0:1787829665.639164:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829665.639166:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829665.639167:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829665.639170:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829665.639172:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829665.639174:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829665.639175:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829665.639178:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829665.639180:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829665.639182:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829665.639183:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829665.639185:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829665.639192:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787829665.639194:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829665.639196:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1787829665.639197:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787829665.639199:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787829665.639200:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787829665.639202:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829665.639204:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787829665.639205:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829665.639208:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829665.639211:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646852608:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829665.639215:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1787829665.639217:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1787829665.639219:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829665.639220:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1787829665.639224:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829665.639231:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787829665.639233:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417869248 : -126234291682368 : ffff8d30c9e2c3c0) 00000100:00000001:3.0:1787829665.639238:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1787829665.639251:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:3.0:1787829665.639255:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787829665.639257:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829665.639258:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829665.639259:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1787829665.639261:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829665.639265:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4299138582 00000100:00000001:3.0:1787829665.639267:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4299139840 00000100:00000001:3.0:1787829665.639269:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4299138582 00000100:00000001:3.0:1787829665.639270:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4299139840 00000100:00000001:3.0:1787829665.639272:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829665.639273:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829665.639274:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1787829665.639277:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829665/real 1787829665] req@ffff8d30c9e2c3c0 x1874670646852608/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829690 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1787829665.639283:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787829665.639283:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829665.639285:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829665.639286:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829665.639287:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829665.639288:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829665.639289:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829665.639292:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2c3c0 x1874670646852608/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829690 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829665.639306:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829665.639307:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829665.639309:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829665.639310:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829665.639311:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829665.639312:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829665.639314:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829665.639315:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829665.639317:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829665.639319:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829665.639321:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646852608:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829665.639323:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787829665.639324:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787829665.639326:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829665.639327:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787829665.639328:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829665.639330:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829665.639332:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2d2c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646852736:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829665.639334:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1787829665.639335:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1787829665.639336:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829665.639338:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1787829665.639340:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829665.639343:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787829665.639344:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417873088 : -126234291678528 : ffff8d30c9e2d2c0) 00000100:00000001:3.0:1787829665.639346:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1787829665.639354:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:3.0:1787829665.639357:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787829665.639358:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829665.639359:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829665.639360:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1787829665.639362:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829665.639364:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4299138582 00000100:00000001:3.0:1787829665.639366:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4299139840 00000100:00000001:3.0:1787829665.639367:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4299138582 00000100:00000001:3.0:1787829665.639369:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4299139840 00000100:00000001:3.0:1787829665.639370:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829665.639372:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829665.639373:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1787829665.639375:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829665/real 1787829665] req@ffff8d30c9e2d2c0 x1874670646852736/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829690 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1787829665.639380:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787829665.639381:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829665.639382:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829665.639383:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829665.639384:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829665.639385:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829665.639386:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829665.639388:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2d2c0 x1874670646852736/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829690 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829665.639393:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829665.639394:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829665.639396:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829665.639396:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829665.639397:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829665.639398:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829665.639399:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829665.639401:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829665.639402:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829665.639403:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829665.639405:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2d2c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646852736:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829665.639407:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787829665.639408:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787829665.639409:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829665.639410:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787829665.639411:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829665.639412:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829665.639414:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646852864:192.168.203.143@tcp:8:kworker.0 00000100:00000001:3.0:1787829665.639416:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1787829665.639417:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1787829665.639418:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829665.639419:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1787829665.639421:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829665.639424:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787829665.639425:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417875968 : -126234291675648 : ffff8d30c9e2de00) 00000100:00000001:3.0:1787829665.639427:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1787829665.639433:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:3.0:1787829665.639436:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787829665.639437:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829665.639438:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829665.639439:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1787829665.639440:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829665.639442:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4299138595 00000100:00000001:3.0:1787829665.639444:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4299139840 00000100:00000001:3.0:1787829665.639445:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4299138595 00000100:00000001:3.0:1787829665.639447:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4299139840 00000100:00000001:3.0:1787829665.639448:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829665.639449:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829665.639450:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1787829665.639453:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829665/real 1787829665] req@ffff8d30c9e2de00 x1874670646852864/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829690 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1787829665.639457:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787829665.639458:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829665.639459:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829665.639460:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829665.639461:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829665.639462:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829665.639462:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829665.639464:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2de00 x1874670646852864/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829690 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829665.639482:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829665.639483:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829665.639485:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829665.639486:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829665.639487:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829665.639488:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829665.639489:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829665.639491:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829665.639493:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829665.639495:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829665.639497:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646852864:192.168.203.143@tcp:8:kworker.0 00000100:00000001:3.0:1787829665.639499:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829665.639500:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829665.639501:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829665.639502:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829665.639504:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829665.639505:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829665.639506:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829665.639508:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829665.639509:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829665.639510:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829665.639511:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829665.639512:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829665.639513:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829665.639514:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829665.639515:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829665.639516:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829665.639517:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829665.639518:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829665.639518:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829665.639520:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829665.639521:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829665.639522:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829665.639523:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829665.639523:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829665.639524:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829665.639525:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829665.639526:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829665.639526:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829665.639527:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829665.639529:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829665.639530:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829665.639531:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829665.639533:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787829665.639534:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829665.639536:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1787829665.639537:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829666.534676:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829666.534683:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829666.534754:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829666.534758:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829666.534761:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829667.558686:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829667.558692:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829667.558749:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829667.558753:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829667.558757:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829668.582668:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829668.582674:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829668.582729:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829668.582733:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829668.582735:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829669.606685:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829669.606690:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829669.606736:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829669.606739:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829669.606742:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829670.630693:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829670.630701:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829670.630751:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829670.630756:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829670.630760:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787829670.758754:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829670.758797:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1787829670.758800:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787829670.758805:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787829670.758811:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787829670.758849:0:40798:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 4471 00000100:00080000:0.0:1787829670.758863:0:40798:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787829670.758867:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829670.758869:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787829670.758875:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787829670.758886:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787829670.758890:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:0.0:1787829670.758893:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:0.0:1787829670.758896:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829670.758897:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:0.0:1787829670.758902:0:40798:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787829670.758904:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829670.758907:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787829670.758911:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787829670.758913:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829670.758916:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1787829670.758920:0:40798:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829670.758946:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787829670.758949:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829670.758953:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787829670.758956:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787829670.758962:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829670.758977:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787829670.758989:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d26940 x1874670646852992/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787829670.759018:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787829670.759024:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829670.759044:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787829670.759048:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829670.759053:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1787829670.759054:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787829670.759056:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787829670.759066:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787829670.759069:0:40798:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4471 00000100:00080000:0.0:1787829670.759074:0:40798:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787829670.759077:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829670.759078:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787829670.759081:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:3.0:1787829670.759082:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:0.0:1787829670.759083:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787829670.759085:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000020:00000001:0.0:1787829670.759088:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:3.0:1787829670.759088:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787829670.759090:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829670.759091:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:3.0:1787829670.759092:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787829670.759095:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00080000:0.0:1787829670.759096:0:40798:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:3.0:1787829670.759098:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829670.759099:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829670.759101:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787829670.759102:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787829670.759106:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787829670.759107:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829670.759109:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829670.759110:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829670.759112:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00100000:3.0:1787829670.759114:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d26940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646852992:192.168.203.143@tcp:250:kworker.0 00000008:00100000:0.0:1787829670.759118:0:40798:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00000001:3.0:1787829670.759120:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000008:00000001:0.0:1787829670.759121:0:40798:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829670.759123:0:40798:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829670.759125:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787829670.759128:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787829670.759128:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829670.759130:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829670.759131:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:0.0:1787829670.759132:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787829670.759133:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787829670.759135:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829670.759137:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829670.759141:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787829670.759146:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d27c00 x1874670646853120/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787829670.759154:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787829670.759156:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829670.759156:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787829670.759158:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601346880 : -126234108204736 : ffff8d30d4d26940) 00000100:00000001:0.0:1787829670.759159:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787829670.759162:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1787829670.759163:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00080000:0.0:1787829670.759166:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1787829670.759168:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787829670.759170:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787829670.759172:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787829670.759174:0:40798:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4471 00000100:00080000:0.0:1787829670.759178:0:40798:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787829670.759195:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829670.759197:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787829670.759199:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787829670.759200:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787829670.759202:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000020:00000001:0.0:1787829670.759204:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:0.0:1787829670.759206:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829670.759207:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:0.0:1787829670.759210:0:40798:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787829670.759212:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829670.759214:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787829670.759216:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787829670.759218:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829670.759220:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1787829670.759223:0:40798:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1787829670.759226:0:40798:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829670.759228:0:40798:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829670.759231:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000800:00000100:3.0:1787829670.759231:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 02000000:00000001:0.0:1787829670.759233:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829670.759234:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787829670.759236:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787829670.759237:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829670.759242:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787829670.759246:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d25680 x1874670646853248/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787829670.759249:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1787829670.759269:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787829670.759270:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787829670.759272:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829670.759273:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829670.759275:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829670.759275:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00080000:0.0:1787829670.759278:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829670.759281:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1787829670.759283:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:3.0:1787829670.759283:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787829670.759284:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787829670.759287:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:3.0:1787829670.759288:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4299144960 00000100:00080000:0.0:1787829670.759290:0:40798:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4471 00000100:00080000:0.0:1787829670.759293:0:40798:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787829670.759296:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829670.759297:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787829670.759299:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:3.0:1787829670.759300:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 00000020:00000001:0.0:1787829670.759301:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787829670.759303:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000020:00000001:0.0:1787829670.759305:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:0.0:1787829670.759307:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829670.759308:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4299144960 00000100:00000001:0.0:1787829670.759309:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:0.0:1787829670.759313:0:40798:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:3.0:1787829670.759313:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1787829670.759316:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829670.759317:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787829670.759318:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787829670.759321:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829670.759321:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829670.759323:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829670.759325:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00000001:0.0:1787829670.759326:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1787829670.759328:0:40798:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787829670.759330:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829670/real 1787829670] req@ffff8d30d4d26940 x1874670646852992/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829695 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00000001:0.0:1787829670.759331:0:40798:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829670.759333:0:40798:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829670.759337:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787829670.759339:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829670.759340:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787829670.759342:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787829670.759343:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:3.0:1787829670.759343:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787829670.759345:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787829670.759347:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829670.759349:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00100000:0.0:1787829670.759352:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d252c0 x1874670646853376/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787829670.759352:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829670.759355:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829670.759358:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787829670.759360:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787829670.759361:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829670.759363:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829670.759365:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829670.759366:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d26940 x1874670646852992/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829695 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829670.759374:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829670.759377:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829670.759380:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829670.759382:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829670.759384:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829670.759387:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829670.759389:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829670.759393:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829670.759397:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829670.759400:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829670.759405:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d26940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646852992:192.168.203.143@tcp:250:kworker.0 00000100:00000001:3.0:1787829670.759411:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829670.759414:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829670.759416:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829670.759420:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829670.759423:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829670.759425:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829670.759427:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829670.759432:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829670.759447:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829670.759451:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829670.759452:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829670.759455:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829670.759463:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787829670.759481:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829670.759483:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1787829670.759484:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787829670.759485:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787829670.759486:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787829670.759487:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829670.759489:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787829670.759490:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829670.759492:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829670.759495:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d27c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646853120:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829670.759497:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1787829670.759499:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1787829670.759500:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829670.759501:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1787829670.759504:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829670.759508:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787829670.759510:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601351680 : -126234108199936 : ffff8d30d4d27c00) 00000100:00000001:3.0:1787829670.759515:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1787829670.759525:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:3.0:1787829670.759528:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787829670.759530:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829670.759531:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829670.759532:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1787829670.759534:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829670.759538:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4299143702 00000100:00000001:3.0:1787829670.759540:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4299144960 00000100:00000001:3.0:1787829670.759542:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4299143702 00000100:00000001:3.0:1787829670.759543:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4299144960 00000100:00000001:3.0:1787829670.759545:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829670.759546:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829670.759547:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1787829670.759550:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829670/real 1787829670] req@ffff8d30d4d27c00 x1874670646853120/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829695 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1787829670.759556:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787829670.759557:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829670.759559:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829670.759560:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829670.759561:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829670.759562:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829670.759563:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829670.759566:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d27c00 x1874670646853120/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829695 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829670.759570:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829670.759572:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829670.759573:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829670.759574:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829670.759575:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829670.759576:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829670.759577:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829670.759579:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829670.759581:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829670.759582:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829670.759584:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d27c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646853120:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829670.759586:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787829670.759587:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787829670.759588:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829670.759590:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787829670.759591:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829670.759592:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829670.759594:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d25680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646853248:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829670.759596:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1787829670.759597:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1787829670.759598:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829670.759599:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1787829670.759602:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829670.759604:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787829670.759605:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601342080 : -126234108209536 : ffff8d30d4d25680) 00000100:00000001:3.0:1787829670.759608:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1787829670.759614:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:3.0:1787829670.759663:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787829670.759665:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829670.759666:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829670.759667:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1787829670.759670:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829670.759673:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4299143703 00000100:00000001:3.0:1787829670.759675:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4299144960 00000100:00000001:3.0:1787829670.759677:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4299143703 00000100:00000001:3.0:1787829670.759678:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4299144960 00000100:00000001:3.0:1787829670.759680:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829670.759682:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829670.759683:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1787829670.759686:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829670/real 1787829670] req@ffff8d30d4d25680 x1874670646853248/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829695 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1787829670.759691:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787829670.759692:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829670.759694:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829670.759695:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829670.759696:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829670.759697:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829670.759698:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829670.759701:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d25680 x1874670646853248/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829695 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829670.759706:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829670.759707:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829670.759710:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829670.759711:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829670.759712:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829670.759713:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829670.759714:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829670.759716:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829670.759718:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829670.759720:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829670.759722:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d25680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646853248:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829670.759724:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787829670.759725:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787829670.759726:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829670.759727:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787829670.759728:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829670.759730:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829670.759731:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d252c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646853376:192.168.203.143@tcp:8:kworker.0 00000100:00000001:3.0:1787829670.759733:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1787829670.759734:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1787829670.759736:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829670.759737:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1787829670.759740:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829670.759753:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787829670.759754:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601341120 : -126234108210496 : ffff8d30d4d252c0) 00000100:00000001:3.0:1787829670.759757:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1787829670.759766:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:3.0:1787829670.759768:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787829670.759770:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829670.759771:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829670.759773:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1787829670.759775:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829670.759777:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4299143716 00000100:00000001:3.0:1787829670.759779:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4299144960 00000100:00000001:3.0:1787829670.759781:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4299143716 00000100:00000001:3.0:1787829670.759783:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4299144960 00000100:00000001:3.0:1787829670.759785:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829670.759786:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829670.759788:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1787829670.759790:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829670/real 1787829670] req@ffff8d30d4d252c0 x1874670646853376/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829695 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1787829670.759795:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787829670.759796:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829670.759798:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829670.759799:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829670.759800:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829670.759802:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829670.759803:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829670.759805:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d252c0 x1874670646853376/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829695 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829670.759826:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829670.759828:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829670.759829:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829670.759835:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829670.759836:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829670.759837:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829670.759838:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829670.759840:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829670.759842:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829670.759844:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829670.759846:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d252c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646853376:192.168.203.143@tcp:8:kworker.0 00000100:00000001:3.0:1787829670.759848:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829670.759849:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829670.759851:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829670.759852:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829670.759853:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829670.759855:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829670.759855:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829670.759857:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829670.759858:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829670.759861:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829670.759862:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829670.759863:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829670.759864:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829670.759865:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829670.759866:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829670.759867:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829670.759868:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829670.759869:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829670.759870:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829670.759871:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829670.759872:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829670.759873:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829670.759873:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829670.759874:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829670.759875:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829670.759876:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829670.759876:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829670.759877:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829670.759878:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829670.759879:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829670.759879:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829670.759880:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829670.759882:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787829670.759883:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829670.759884:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1787829670.759885:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829671.654676:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829671.654683:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829671.654733:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829671.654736:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829671.654739:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829672.678691:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829672.678698:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829672.678750:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829672.678754:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829672.678757:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829673.702745:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829673.702757:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829673.702905:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829673.702910:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829673.702913:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829674.726730:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829674.726737:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829674.726828:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829674.726833:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829674.726836:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829675.750749:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829675.750758:0:815:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1787829675.750760:0:815:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1787829675.750763:0:815:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829675.750768:0:815:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1787829675.750770:0:815:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829675.750772:0:815:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829675.750773:0:815:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829675.750848:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829675.750853:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829675.750857:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787829675.878729:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829675.878740:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1787829675.878743:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787829675.878746:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787829675.878751:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787829675.878760:0:40798:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 4476 00000100:00080000:2.0:1787829675.878772:0:40798:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787829675.878775:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829675.878778:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787829675.878783:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787829675.878786:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787829675.878789:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:2.0:1787829675.878792:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:2.0:1787829675.878793:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829675.878794:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:2.0:1787829675.878799:0:40798:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787829675.878802:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829675.878806:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787829675.878809:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787829675.878812:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829675.878814:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:2.0:1787829675.878817:0:40798:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829675.878829:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787829675.878831:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829675.878835:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787829675.878837:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787829675.878842:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829675.878853:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787829675.878862:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2e1c0 x1874670646853504/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787829675.878870:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787829675.878873:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829675.878888:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787829675.878891:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829675.878893:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1787829675.878894:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787829675.878895:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787829675.878898:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787829675.878900:0:40798:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4476 00000100:00080000:2.0:1787829675.878903:0:40798:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787829675.878904:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829675.878906:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787829675.878908:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787829675.878910:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787829675.878912:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000020:00000001:2.0:1787829675.878915:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:2.0:1787829675.878918:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829675.878920:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787829675.878921:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:3.0:1787829675.878926:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00080000:2.0:1787829675.878928:0:40798:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:3.0:1787829675.878930:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1787829675.878934:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829675.878934:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787829675.878938:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829675.878940:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787829675.878942:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787829675.878945:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829675.878947:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829675.878952:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646853504:192.168.203.143@tcp:250:kworker.0 02000000:00000001:2.0:1787829675.878956:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829675.878957:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1787829675.878961:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1787829675.878965:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829675.878965:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829675.878970:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 02000000:00000001:3.0:1787829675.878971:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000008:00100000:2.0:1787829675.878975:0:40798:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 02000000:00000001:3.0:1787829675.878976:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:2.0:1787829675.878980:0:40798:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829675.878983:0:40798:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829675.878988:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787829675.878989:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829675.878990:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787829675.878991:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:3.0:1787829675.878992:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 02000000:00000001:2.0:1787829675.878993:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829675.878995:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417876928 : -126234291674688 : ffff8d30c9e2e1c0) 00000100:00000001:3.0:1787829675.878999:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:2.0:1787829675.879000:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787829675.879007:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2d680 x1874670646853632/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787829675.879013:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787829675.879015:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829675.879018:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787829675.879020:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829675.879024:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1787829675.879025:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787829675.879026:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787829675.879028:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787829675.879030:0:40798:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4476 00000100:00080000:2.0:1787829675.879033:0:40798:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787829675.879036:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000800:00000100:3.0:1787829675.879037:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:2.0:1787829675.879038:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787829675.879042:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787829675.879043:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787829675.879045:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000020:00000001:2.0:1787829675.879046:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:2.0:1787829675.879048:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829675.879050:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:2.0:1787829675.879053:0:40798:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:3.0:1787829675.879054:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1787829675.879057:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829675.879060:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:3.0:1787829675.879061:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 02000000:00000001:2.0:1787829675.879064:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787829675.879068:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829675.879070:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:3.0:1787829675.879071:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000008:00100000:2.0:1787829675.879073:0:40798:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00000001:3.0:1787829675.879076:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000008:00000001:2.0:1787829675.879078:0:40798:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829675.879082:0:40798:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829675.879085:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829675.879089:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:3.0:1787829675.879090:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4299150080 02000000:00000001:2.0:1787829675.879092:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829675.879097:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787829675.879099:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787829675.879100:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829675.879105:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 00000100:00000001:2.0:1787829675.879107:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829675.879123:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4299150080 00000100:00100000:2.0:1787829675.879127:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2de00 x1874670646853760/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787829675.879131:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 00000100:00000001:3.0:1787829675.879141:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829675.879145:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829675.879150:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00000001:2.0:1787829675.879152:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787829675.879156:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829675.879160:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829675.879161:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829675/real 1787829675] req@ffff8d30c9e2e1c0 x1874670646853504/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829700 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829675.879163:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829675.879170:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1787829675.879174:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787829675.879177:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787829675.879180:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000400:00000001:3.0:1787829675.879181:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00080000:2.0:1787829675.879184:0:40798:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4476 00000400:00000001:3.0:1787829675.879185:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829675.879190:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00080000:2.0:1787829675.879193:0:40798:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787829675.879194:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829675.879197:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829675.879199:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829675.879200:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787829675.879204:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:3.0:1787829675.879204:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:2.0:1787829675.879210:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000100:00100000:3.0:1787829675.879211:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2e1c0 x1874670646853504/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829700 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:2.0:1787829675.879214:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787829675.879217:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:3.0:1787829675.879221:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:2.0:1787829675.879223:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:3.0:1787829675.879225:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787829675.879228:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829675.879229:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829675.879232:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:3.0:1787829675.879233:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829675.879234:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829675.879239:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00080000:2.0:1787829675.879242:0:40798:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:3.0:1787829675.879242:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829675.879245:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00000001:2.0:1787829675.879246:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829675.879249:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00080000:3.0:1787829675.879252:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 02000000:00000001:2.0:1787829675.879253:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787829675.879257:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829675.879257:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000001:2.0:1787829675.879261:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1787829675.879264:0:40798:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00100000:3.0:1787829675.879266:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2e1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646853504:192.168.203.143@tcp:250:kworker.0 00000008:00000001:2.0:1787829675.879267:0:40798:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829675.879270:0:40798:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829675.879276:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:3.0:1787829675.879277:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:2.0:1787829675.879278:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829675.879284:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:3.0:1787829675.879284:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829675.879289:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787829675.879290:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787829675.879294:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829675.879295:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829675.879300:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00000001:2.0:1787829675.879303:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829675.879305:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829675.879308:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829675.879312:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 00000100:00100000:2.0:1787829675.879313:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2d2c0 x1874670646853888/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:3.0:1787829675.879314:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829675.879319:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829675.879320:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829675.879324:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829675.879332:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787829675.879333:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1787829675.879335:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829675.879336:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829675.879341:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829675.879341:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1787829675.879344:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787829675.879346:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787829675.879348:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787829675.879349:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829675.879352:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787829675.879354:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829675.879355:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829675.879359:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2d680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646853632:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829675.879362:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1787829675.879364:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1787829675.879365:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829675.879367:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1787829675.879369:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829675.879374:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787829675.879375:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417874048 : -126234291677568 : ffff8d30c9e2d680) 00000100:00000001:3.0:1787829675.879379:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1787829675.879390:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:3.0:1787829675.879395:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787829675.879410:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829675.879412:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829675.879414:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1787829675.879417:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829675.879422:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4299148822 00000100:00000001:3.0:1787829675.879425:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4299150080 00000100:00000001:3.0:1787829675.879428:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4299148822 00000100:00000001:3.0:1787829675.879430:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4299150080 00000100:00000001:3.0:1787829675.879432:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829675.879434:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829675.879436:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1787829675.879441:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829675/real 1787829675] req@ffff8d30c9e2d680 x1874670646853632/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829700 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1787829675.879450:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787829675.879451:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829675.879453:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829675.879455:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829675.879456:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829675.879458:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829675.879460:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829675.879464:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2d680 x1874670646853632/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829700 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829675.879471:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829675.879473:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829675.879475:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829675.879477:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829675.879479:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829675.879480:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829675.879482:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829675.879485:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829675.879488:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829675.879491:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829675.879494:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2d680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646853632:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829675.879497:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787829675.879498:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787829675.879500:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829675.879502:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787829675.879504:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829675.879505:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829675.879508:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646853760:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829675.879512:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1787829675.879513:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1787829675.879515:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829675.879516:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1787829675.879519:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829675.879523:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787829675.879525:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417875968 : -126234291675648 : ffff8d30c9e2de00) 00000100:00000001:3.0:1787829675.879528:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1787829675.879541:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:3.0:1787829675.879545:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787829675.879548:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829675.879550:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829675.879552:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1787829675.879554:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829675.879558:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4299148822 00000100:00000001:3.0:1787829675.879560:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4299150080 00000100:00000001:3.0:1787829675.879562:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4299148822 00000100:00000001:3.0:1787829675.879565:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4299150080 00000100:00000001:3.0:1787829675.879567:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829675.879569:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829675.879571:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1787829675.879576:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829675/real 1787829675] req@ffff8d30c9e2de00 x1874670646853760/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829700 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1787829675.879584:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787829675.879586:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829675.879588:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829675.879589:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829675.879590:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829675.879591:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829675.879599:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829675.879623:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2de00 x1874670646853760/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829700 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829675.879628:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829675.879630:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829675.879631:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829675.879632:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829675.879633:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829675.879634:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829675.879635:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829675.879637:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829675.879638:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829675.879640:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829675.879642:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2de00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646853760:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829675.879645:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787829675.879646:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787829675.879647:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829675.879648:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787829675.879649:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829675.879651:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829675.879653:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2d2c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646853888:192.168.203.143@tcp:8:kworker.0 00000100:00000001:3.0:1787829675.879655:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1787829675.879656:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1787829675.879657:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829675.879658:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1787829675.879661:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829675.879665:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787829675.879666:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417873088 : -126234291678528 : ffff8d30c9e2d2c0) 00000100:00000001:3.0:1787829675.879669:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1787829675.879678:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:3.0:1787829675.879681:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787829675.879682:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829675.879683:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829675.879684:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1787829675.879686:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829675.879689:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4299148836 00000100:00000001:3.0:1787829675.879691:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4299150080 00000100:00000001:3.0:1787829675.879692:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4299148836 00000100:00000001:3.0:1787829675.879693:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4299150080 00000100:00000001:3.0:1787829675.879695:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829675.879696:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829675.879697:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1787829675.879699:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829675/real 1787829675] req@ffff8d30c9e2d2c0 x1874670646853888/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829700 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1787829675.879709:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787829675.879710:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829675.879712:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829675.879713:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829675.879714:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829675.879715:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829675.879716:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829675.879719:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2d2c0 x1874670646853888/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829700 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829675.879723:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829675.879724:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829675.879726:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829675.879727:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829675.879727:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829675.879728:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829675.879729:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829675.879731:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829675.879733:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829675.879734:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829675.879736:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2d2c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646853888:192.168.203.143@tcp:8:kworker.0 00000100:00000001:3.0:1787829675.879738:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829675.879739:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829675.879740:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829675.879741:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829675.879743:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829675.879744:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829675.879745:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829675.879746:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829675.879747:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829675.879750:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829675.879751:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829675.879753:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829675.879753:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829675.879754:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829675.879755:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829675.879756:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829675.879757:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829675.879758:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829675.879759:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829675.879760:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829675.879761:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829675.879762:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829675.879763:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829675.879763:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829675.879764:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829675.879765:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829675.879766:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829675.879767:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829675.879767:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829675.879768:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829675.879769:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829675.879770:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829675.879772:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787829675.879773:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829675.879774:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1787829675.879775:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829676.774693:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829676.774701:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829676.774780:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829676.774788:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829676.774792:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829677.798684:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829677.798690:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829677.798724:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829677.798727:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829677.798730:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829678.822684:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829678.822689:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829678.822728:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829678.822732:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829678.822735:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829679.846690:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829679.846694:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829679.846753:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829679.846757:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829679.846761:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829680.870709:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829680.870717:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829680.870777:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829680.870782:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829680.870785:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787829680.998721:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829680.998731:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1787829680.998734:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1787829680.998737:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787829680.998742:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1787829680.998749:0:468:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 4481 00000100:00080000:1.0:1787829680.998760:0:468:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1787829680.998763:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787829680.998766:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:1.0:1787829680.998770:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:1.0:1787829680.998772:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:1.0:1787829680.998776:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:1.0:1787829680.998779:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:1.0:1787829680.998781:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787829680.998783:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:1.0:1787829680.998787:0:468:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:1.0:1787829680.998791:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829680.998794:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1787829680.998797:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829680.998800:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829680.998803:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:1.0:1787829680.998806:0:468:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829680.998817:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1787829680.998819:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829680.998823:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1787829680.998825:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1787829680.998829:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829680.998841:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829680.998850:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d13efc00 x1874670646854016/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787829680.998862:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787829680.998866:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829680.998880:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1787829680.998884:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829680.998887:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1787829680.998889:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1787829680.998891:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787829680.998894:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1787829680.998897:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4481 00000100:00080000:1.0:1787829680.998902:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1787829680.998904:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787829680.998906:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:1.0:1787829680.998910:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:3.0:1787829680.998911:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:1.0:1787829680.998913:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:3.0:1787829680.998916:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000020:00000001:1.0:1787829680.998917:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:3.0:1787829680.998919:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000020:00000001:1.0:1787829680.998923:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:3.0:1787829680.998924:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787829680.998927:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829680.998928:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787829680.998932:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 02000000:00000001:3.0:1787829680.998932:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787829680.998936:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00080000:1.0:1787829680.998940:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 02000000:00000001:3.0:1787829680.998941:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829680.998947:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829680.998949:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d13efc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646854016:192.168.203.143@tcp:250:kworker.0 02000000:00000001:1.0:1787829680.998951:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:3.0:1787829680.998956:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787829680.998959:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829680.998960:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1787829680.998963:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829680.998965:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829680.998967:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:1.0:1787829680.998969:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 02000000:00000001:3.0:1787829680.998973:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000008:00100000:1.0:1787829680.998975:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1787829680.998980:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829680.998983:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829680.998987:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 02000000:00000001:1.0:1787829680.998988:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:3.0:1787829680.998990:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839541353472 : -126234168198144 : ffff8d30d13efc00) 02000000:00000001:1.0:1787829680.998992:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829680.998995:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:1.0:1787829680.998996:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1787829680.998999:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1787829680.999002:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829680.999008:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829680.999015:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d13ede00 x1874670646854144/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787829680.999025:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787829680.999027:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:3.0:1787829680.999032:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787829680.999033:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1787829680.999038:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829680.999042:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1787829680.999044:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787829680.999045:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1787829680.999049:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787829680.999050:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829680.999054:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:3.0:1787829680.999054:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829680.999058:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00080000:1.0:1787829680.999061:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4481 00000100:00000001:3.0:1787829680.999063:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1787829680.999067:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787829680.999068:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4299155200 00000100:00000001:1.0:1787829680.999073:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787829680.999076:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:1.0:1787829680.999079:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:3.0:1787829680.999079:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 00000020:00000001:1.0:1787829680.999083:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:1.0:1787829680.999086:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000020:00000001:1.0:1787829680.999089:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:1.0:1787829680.999092:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787829680.999093:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:3.0:1787829680.999097:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4299155200 00000100:00080000:1.0:1787829680.999098:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:1.0:1787829680.999104:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829680.999106:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 02000000:00000001:1.0:1787829680.999107:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1787829680.999112:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829680.999114:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829680.999117:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829680.999119:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829680.999121:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:3.0:1787829680.999124:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000008:00100000:1.0:1787829680.999125:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1787829680.999130:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829680.999133:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829680.999135:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829680/real 1787829680] req@ffff8d30d13efc00 x1874670646854016/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829705 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:1.0:1787829680.999138:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1787829680.999141:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829680.999146:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1787829680.999149:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1787829680.999151:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:3.0:1787829680.999151:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787829680.999154:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787829680.999158:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829680.999160:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829680.999165:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00100000:1.0:1787829680.999167:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d13eed00 x1874670646854272/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787829680.999168:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829680.999174:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829680.999177:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829680.999185:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00100000:3.0:1787829680.999188:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d13efc00 x1874670646854016/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829705 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787829680.999189:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829680.999193:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1787829680.999198:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829680.999207:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1787829680.999210:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829680.999211:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:3.0:1787829680.999214:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787829680.999215:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787829680.999220:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829680.999221:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:3.0:1787829680.999223:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829680.999227:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:1.0:1787829680.999228:0:468:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4481 00000100:00000001:3.0:1787829680.999229:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829680.999232:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787829680.999236:0:468:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829680.999236:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00000001:1.0:1787829680.999243:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00080000:3.0:1787829680.999244:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787829680.999246:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:3.0:1787829680.999248:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000020:00000001:1.0:1787829680.999253:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000100:00100000:3.0:1787829680.999255:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d13efc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646854016:192.168.203.143@tcp:250:kworker.0 00000020:00000001:1.0:1787829680.999257:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:3.0:1787829680.999260:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:1.0:1787829680.999261:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:3.0:1787829680.999264:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000020:00000001:1.0:1787829680.999267:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:3.0:1787829680.999267:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829680.999271:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 00000100:00000001:1.0:1787829680.999272:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787829680.999275:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 02000000:00000001:3.0:1787829680.999275:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829680.999279:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829680.999281:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 00000100:00080000:1.0:1787829680.999284:0:468:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 02000000:00000001:3.0:1787829680.999287:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 00000100:00000001:1.0:1787829680.999289:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829680.999290:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829680.999293:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 02000000:00000001:1.0:1787829680.999294:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:3.0:1787829680.999296:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829680.999298:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:1.0:1787829680.999300:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829680.999304:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829680.999305:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787829680.999309:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:3.0:1787829680.999309:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829680.999314:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000008:00100000:1.0:1787829680.999315:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00000001:3.0:1787829680.999316:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787829680.999318:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787829680.999320:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000008:00000001:1.0:1787829680.999322:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829680.999323:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829680.999328:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829680.999329:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787829680.999332:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829680.999336:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787829680.999336:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829680.999340:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829680.999340:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d13ede00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646854144:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787829680.999346:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:3.0:1787829680.999346:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787829680.999349:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787829680.999350:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787829680.999353:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829680.999353:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829680.999360:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1787829680.999374:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829680.999376:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829680.999384:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00100000:1.0:1787829680.999385:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d13ef840 x1874670646854400/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787829680.999388:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839541345792 : -126234168205824 : ffff8d30d13ede00) 00000100:00000001:3.0:1787829680.999397:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:1.0:1787829680.999399:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787829680.999403:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829680.999407:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:3.0:1787829680.999418:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:3.0:1787829680.999424:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787829680.999428:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829680.999429:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829680.999431:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1787829680.999434:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829680.999438:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4299153942 00000100:00000001:3.0:1787829680.999440:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4299155200 00000100:00000001:3.0:1787829680.999442:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4299153942 00000100:00000001:3.0:1787829680.999444:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4299155200 00000100:00000001:3.0:1787829680.999445:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829680.999447:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829680.999448:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1787829680.999451:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829680/real 1787829680] req@ffff8d30d13ede00 x1874670646854144/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829705 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1787829680.999458:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787829680.999459:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829680.999460:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829680.999461:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829680.999462:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829680.999464:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829680.999465:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829680.999468:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d13ede00 x1874670646854144/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829705 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829680.999472:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829680.999474:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829680.999475:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829680.999476:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829680.999477:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829680.999479:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829680.999486:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829680.999487:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829680.999490:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829680.999491:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829680.999494:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d13ede00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646854144:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829680.999496:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787829680.999497:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787829680.999498:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829680.999499:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787829680.999501:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829680.999502:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829680.999504:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d13eed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646854272:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829680.999506:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1787829680.999508:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1787829680.999509:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829680.999510:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1787829680.999512:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829680.999515:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787829680.999516:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839541349632 : -126234168201984 : ffff8d30d13eed00) 00000100:00000001:3.0:1787829680.999519:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1787829680.999527:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:3.0:1787829680.999530:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787829680.999532:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829680.999533:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829680.999534:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1787829680.999536:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829680.999538:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4299153942 00000100:00000001:3.0:1787829680.999539:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4299155200 00000100:00000001:3.0:1787829680.999541:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4299153942 00000100:00000001:3.0:1787829680.999542:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4299155200 00000100:00000001:3.0:1787829680.999545:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829680.999546:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829680.999547:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1787829680.999550:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829680/real 1787829680] req@ffff8d30d13eed00 x1874670646854272/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829705 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1787829680.999555:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787829680.999556:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829680.999557:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829680.999558:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829680.999559:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829680.999560:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829680.999561:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829680.999563:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d13eed00 x1874670646854272/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829705 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829680.999568:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829680.999569:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829680.999571:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829680.999572:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829680.999573:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829680.999574:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829680.999575:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829680.999576:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829680.999586:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829680.999588:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829680.999590:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d13eed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646854272:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829680.999592:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829680.999593:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829680.999594:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829680.999595:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829680.999597:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829680.999598:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829680.999599:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829680.999630:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829680.999632:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829680.999634:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829680.999635:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829680.999636:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829680.999637:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829680.999638:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829680.999639:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829680.999640:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829680.999641:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829680.999642:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829680.999643:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829680.999644:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829680.999645:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829680.999646:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829680.999648:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787829680.999649:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829680.999650:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1787829680.999651:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787829680.999652:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787829680.999653:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787829680.999654:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829680.999656:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787829680.999657:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829680.999659:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829680.999661:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d13ef840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646854400:192.168.203.143@tcp:8:kworker.0 00000100:00000001:3.0:1787829680.999663:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1787829680.999665:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1787829680.999666:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829680.999667:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1787829680.999669:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829680.999673:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787829680.999674:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839541352512 : -126234168199104 : ffff8d30d13ef840) 00000100:00000001:3.0:1787829680.999677:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1787829680.999685:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:3.0:1787829680.999688:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787829680.999689:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829680.999690:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829680.999692:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1787829680.999694:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829680.999696:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4299153956 00000100:00000001:3.0:1787829680.999698:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4299155200 00000100:00000001:3.0:1787829680.999706:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4299153956 00000100:00000001:3.0:1787829680.999708:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4299155200 00000100:00000001:3.0:1787829680.999710:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829680.999711:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829680.999712:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1787829680.999715:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829680/real 1787829680] req@ffff8d30d13ef840 x1874670646854400/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829705 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1787829680.999721:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787829680.999722:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829680.999724:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829680.999725:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829680.999726:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829680.999727:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829680.999728:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829680.999730:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d13ef840 x1874670646854400/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829705 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829680.999735:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829680.999736:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829680.999738:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829680.999739:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829680.999739:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829680.999740:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829680.999742:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829680.999743:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829680.999746:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829680.999747:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829680.999750:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d13ef840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646854400:192.168.203.143@tcp:8:kworker.0 00000100:00000001:3.0:1787829680.999753:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829680.999754:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829680.999755:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829680.999756:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829680.999757:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829680.999758:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829680.999758:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829680.999760:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829680.999761:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829680.999762:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829680.999763:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829680.999764:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829680.999766:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787829680.999767:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829680.999768:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1787829680.999769:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829681.894664:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829681.894671:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829681.894738:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829681.894743:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829681.894747:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829682.918698:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829682.918704:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829682.918757:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829682.918760:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829682.918763:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829683.942693:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829683.942700:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829683.942757:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829683.942760:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829683.942763:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829684.966714:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829684.966720:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829684.966776:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829684.966780:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829684.966784:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829685.990686:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829685.990693:0:815:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1787829685.990695:0:815:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1787829685.990698:0:815:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829685.990703:0:815:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1787829685.990705:0:815:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829685.990707:0:815:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829685.990708:0:815:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829685.990761:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829685.990765:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829685.990769:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787829686.118728:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829686.118735:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1787829686.118738:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787829686.118741:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787829686.118745:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787829686.118751:0:468:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 4486 00000100:00080000:3.0:1787829686.118760:0:468:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787829686.118763:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829686.118764:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:3.0:1787829686.118768:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1787829686.118771:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1787829686.118773:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:3.0:1787829686.118776:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:3.0:1787829686.118777:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829686.118779:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:3.0:1787829686.118783:0:468:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:3.0:1787829686.118787:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829686.118791:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787829686.118795:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829686.118799:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829686.118802:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:3.0:1787829686.118805:0:468:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829686.118814:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787829686.118816:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829686.118820:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787829686.118824:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787829686.118828:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829686.118839:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829686.118847:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c7015680 x1874670646854528/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787829686.118858:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787829686.118864:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829686.118876:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787829686.118880:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1787829686.118885:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00080000:3.0:1787829686.118885:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1787829686.118888:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:1.0:1787829686.118892:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00080000:3.0:1787829686.118892:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787829686.118897:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787829686.118899:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:1.0:1787829686.118900:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787829686.118904:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829686.118909:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000100:00080000:3.0:1787829686.118909:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4486 02000000:00000001:1.0:1787829686.118913:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00080000:3.0:1787829686.118916:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 02000000:00000001:1.0:1787829686.118917:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829686.118922:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00100000:1.0:1787829686.118924:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c7015680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646854528:192.168.203.143@tcp:250:kworker.0 00000100:00000001:3.0:1787829686.118926:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:1.0:1787829686.118931:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000020:00000001:3.0:1787829686.118932:0:468:0:(genops.c:792:class_conn2export()) Process entered 02000000:00000001:1.0:1787829686.118936:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000020:00000001:3.0:1787829686.118936:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 02000000:00000001:1.0:1787829686.118951:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829686.118951:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 02000000:00000001:1.0:1787829686.118955:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000020:00000001:3.0:1787829686.118957:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 02000000:00000001:1.0:1787829686.118960:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829686.118962:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829686.118966:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:1.0:1787829686.118975:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787829686.118978:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369541248 : -126234340010368 : ffff8d30c7015680) 00000100:00000001:1.0:1787829686.118981:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1787829686.119012:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787829686.119021:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787829686.119024:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829686.119026:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829686.119027:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00080000:3.0:1787829686.119032:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:1.0:1787829686.119033:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829686.119037:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4299160320 00000100:00000001:3.0:1787829686.119039:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829686.119044:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1787829686.119046:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 02000000:00000001:3.0:1787829686.119050:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829686.119052:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4299160320 02000000:00000001:3.0:1787829686.119055:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829686.119057:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 00000100:00000001:3.0:1787829686.119061:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:1.0:1787829686.119062:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829686.119066:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829686.119068:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000008:00100000:3.0:1787829686.119069:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829686.119073:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829686/real 1787829686] req@ffff8d30c7015680 x1874670646854528/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829711 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787829686.119083:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787829686.119084:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787829686.119086:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787829686.119088:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787829686.119090:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829686.119092:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829686.119093:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787829686.119096:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c7015680 x1874670646854528/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829711 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787829686.119101:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829686.119104:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787829686.119106:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829686.119107:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829686.119108:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829686.119110:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787829686.119111:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787829686.119113:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829686.119116:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787829686.119118:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787829686.119120:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c7015680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646854528:192.168.203.143@tcp:250:kworker.0 00000100:00000001:1.0:1787829686.119123:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829686.119125:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829686.119126:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787829686.119128:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787829686.119130:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787829686.119132:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787829686.119134:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787829686.119138:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787829686.119139:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787829686.119142:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787829686.119142:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829686.119144:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829686.119161:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787829686.119163:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829686.119164:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787829686.119165:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:3.0:1787829686.119189:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829686.119194:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829686.119199:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787829686.119200:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829686.119203:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787829686.119204:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787829686.119207:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829686.119214:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829686.119231:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c7014b40 x1874670646854656/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787829686.119241:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787829686.119244:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829686.119295:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787829686.119298:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829686.119300:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1787829686.119301:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787829686.119303:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787829686.119305:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787829686.119308:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4486 00000100:00080000:3.0:1787829686.119312:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787829686.119315:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829686.119316:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:1.0:1787829686.119318:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:3.0:1787829686.119318:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1787829686.119322:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:1.0:1787829686.119323:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000020:00000001:3.0:1787829686.119324:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:1.0:1787829686.119336:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787829686.119338:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787829686.119339:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829686.119341:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787829686.119344:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000020:00000001:3.0:1787829686.119345:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 02000000:00000001:1.0:1787829686.119347:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829686.119348:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829686.119351:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00100000:1.0:1787829686.119353:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c7014b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646854656:192.168.203.143@tcp:38:kworker.0 00000100:00080000:3.0:1787829686.119357:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:1.0:1787829686.119360:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:3.0:1787829686.119361:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829686.119363:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1787829686.119364:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1787829686.119366:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829686.119369:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829686.119380:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1787829686.119382:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829686.119383:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829686.119390:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787829686.119391:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369538368 : -126234340013248 : ffff8d30c7014b40) 00000100:00000001:3.0:1787829686.119393:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:1.0:1787829686.119395:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000008:00100000:3.0:1787829686.119401:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1787829686.119407:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:1.0:1787829686.119409:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:3.0:1787829686.119412:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829686.119415:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787829686.119418:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829686.119420:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829686.119421:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:1.0:1787829686.119422:0:2390:0:(events.c:68:request_out_callback()) Process leaving 02000000:00000001:3.0:1787829686.119426:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829686.119427:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829686.119432:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:1.0:1787829686.119434:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4299159062 02000000:00000001:3.0:1787829686.119435:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:1.0:1787829686.119439:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4299160320 02000000:00000001:3.0:1787829686.119440:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829686.119445:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4299159062 00000100:00000001:1.0:1787829686.119449:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4299160320 00000100:00000001:3.0:1787829686.119449:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829686.119451:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829686.119454:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829686.119457:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1787829686.119458:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c7014f00 x1874670646854784/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787829686.119464:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829686/real 1787829686] req@ffff8d30c7014b40 x1874670646854656/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829711 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829686.119476:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000400:00000001:1.0:1787829686.119479:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000001:3.0:1787829686.119481:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:1.0:1787829686.119492:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787829686.119496:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829686.119497:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829686.119499:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787829686.119503:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:3.0:1787829686.119504:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1787829686.119505:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829686.119508:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787829686.119513:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c7014b40 x1874670646854656/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829711 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787829686.119515:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1787829686.119520:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787829686.119522:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787829686.119524:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829686.119528:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829686.119528:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:1.0:1787829686.119532:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:3.0:1787829686.119534:0:468:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4486 00000100:00000001:1.0:1787829686.119535:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829686.119537:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829686.119540:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00080000:3.0:1787829686.119541:0:468:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1787829686.119542:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829686.119546:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00080000:1.0:1787829686.119548:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00000001:3.0:1787829686.119549:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:3.0:1787829686.119553:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000100:00080000:1.0:1787829686.119554:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000020:00000001:3.0:1787829686.119556:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:1.0:1787829686.119558:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000020:00000001:3.0:1787829686.119559:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00100000:1.0:1787829686.119563:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c7014b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646854656:192.168.203.143@tcp:38:kworker.0 00000020:00000001:3.0:1787829686.119564:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:3.0:1787829686.119570:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787829686.119571:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829686.119573:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:1.0:1787829686.119574:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829686.119577:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 00000100:00080000:3.0:1787829686.119579:0:468:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 02000000:00000001:1.0:1787829686.119580:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787829686.119583:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787829686.119586:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000100:00000001:3.0:1787829686.119587:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829686.119588:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787829686.119591:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829686.119592:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1787829686.119593:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787829686.119597:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787829686.119616:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829686.119619:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829686.119622:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787829686.119624:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829686.119626:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787829686.119628:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1787829686.119630:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787829686.119632:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787829686.119633:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829686.119635:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787829686.119636:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829686.119638:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829686.119641:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c7014f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646854784:192.168.203.143@tcp:38:kworker.0 02000000:00000001:3.0:1787829686.119641:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829686.119648:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1787829686.119648:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829686.119652:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 02000000:00000001:1.0:1787829686.119653:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787829686.119656:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000008:00100000:3.0:1787829686.119657:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 02000000:00000001:1.0:1787829686.119659:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000008:00000001:3.0:1787829686.119661:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829686.119663:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829686.119664:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829686.119671:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 02000000:00000001:3.0:1787829686.119671:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:1.0:1787829686.119675:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369539328 : -126234340012288 : ffff8d30c7014f00) 02000000:00000001:3.0:1787829686.119676:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829686.119679:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:3.0:1787829686.119681:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787829686.119685:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787829686.119689:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:1.0:1787829686.119692:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787829686.119697:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787829686.119699:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829686.119700:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829686.119703:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829686.119705:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787829686.119707:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829686.119710:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c70161c0 x1874670646854912/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787829686.119711:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4299159063 00000100:00000001:1.0:1787829686.119715:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4299160320 00000100:00000001:1.0:1787829686.119719:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4299159063 00000100:00000001:1.0:1787829686.119723:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4299160320 00000100:00000001:1.0:1787829686.119727:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829686.119731:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829686.119732:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787829686.119733:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00000001:3.0:1787829686.119736:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829686.119740:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829686/real 1787829686] req@ffff8d30c7014f00 x1874670646854784/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829711 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829686.119742:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:1.0:1787829686.119751:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787829686.119752:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787829686.119754:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787829686.119756:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787829686.119757:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829686.119758:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829686.119760:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787829686.119763:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c7014f00 x1874670646854784/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829711 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787829686.119769:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829686.119771:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787829686.119775:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829686.119777:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829686.119779:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829686.119780:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787829686.119782:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787829686.119785:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829686.119789:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787829686.119791:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787829686.119794:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c7014f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646854784:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787829686.119798:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829686.119800:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829686.119802:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787829686.119803:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787829686.119806:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787829686.119807:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787829686.119809:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787829686.119811:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787829686.119812:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787829686.119814:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787829686.119816:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829686.119818:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829686.119820:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787829686.119822:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829686.119824:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787829686.119826:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1787829686.119827:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787829686.119828:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787829686.119829:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829686.119831:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787829686.119832:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829686.119834:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829686.119837:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c70161c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646854912:192.168.203.143@tcp:8:kworker.0 00000100:00000001:1.0:1787829686.119841:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787829686.119843:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787829686.119845:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829686.119846:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787829686.119849:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829686.119856:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787829686.119858:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369544128 : -126234340007488 : ffff8d30c70161c0) 00000100:00000001:1.0:1787829686.119862:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1787829686.119874:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787829686.119879:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787829686.119882:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829686.119883:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829686.119885:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787829686.119887:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829686.119891:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4299159076 00000100:00000001:1.0:1787829686.119893:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4299160320 00000100:00000001:1.0:1787829686.119894:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4299159076 00000100:00000001:1.0:1787829686.119895:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4299160320 00000100:00000001:1.0:1787829686.119897:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829686.119898:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829686.119900:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787829686.119903:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829686/real 1787829686] req@ffff8d30c70161c0 x1874670646854912/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829711 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787829686.119908:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787829686.119908:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787829686.119910:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787829686.119911:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787829686.119912:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829686.119913:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829686.119914:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787829686.119917:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c70161c0 x1874670646854912/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829711 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787829686.119921:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829686.119923:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787829686.119924:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829686.119925:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829686.119926:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829686.119927:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787829686.119928:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787829686.119930:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829686.119931:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787829686.119933:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787829686.119935:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c70161c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646854912:192.168.203.143@tcp:8:kworker.0 00000100:00000001:1.0:1787829686.119937:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829686.119938:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829686.119939:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787829686.119940:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787829686.119941:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787829686.119942:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787829686.119943:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787829686.119944:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787829686.119945:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787829686.119946:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787829686.119947:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829686.119949:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829686.119951:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787829686.119952:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829686.119953:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787829686.119954:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829687.014696:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829687.014701:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829687.014797:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829687.014800:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829687.014803:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829688.038683:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829688.038689:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829688.038736:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829688.038740:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829688.038743:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829689.062695:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829689.062700:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829689.062807:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829689.062812:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829689.062816:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829690.086678:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829690.086683:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829690.086751:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829690.086755:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829690.086759:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829691.110761:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829691.110770:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829691.110887:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829691.110895:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829691.110901:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787829691.238723:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829691.238735:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1787829691.238738:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787829691.238742:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787829691.238748:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787829691.238755:0:468:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 4491 00000100:00080000:3.0:1787829691.238767:0:468:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787829691.238770:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829691.238772:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:3.0:1787829691.238777:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1787829691.238780:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1787829691.238784:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:3.0:1787829691.238787:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:3.0:1787829691.238790:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829691.238792:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:3.0:1787829691.238797:0:468:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:3.0:1787829691.238801:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829691.238804:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787829691.238808:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829691.238811:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829691.238815:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:3.0:1787829691.238818:0:468:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829691.238841:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787829691.238843:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829691.238847:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787829691.238850:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787829691.238854:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829691.238866:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829691.238878:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c7016580 x1874670646855040/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787829691.238892:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787829691.238897:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829691.238915:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787829691.238919:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829691.238923:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1787829691.238925:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787829691.238927:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787829691.238931:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787829691.238934:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4491 00000100:00080000:3.0:1787829691.238955:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787829691.238959:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829691.238962:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:3.0:1787829691.238966:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1787829691.238969:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1787829691.238973:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000020:00000001:3.0:1787829691.238976:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:1.0:1787829691.238980:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1787829691.238980:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829691.238987:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:1.0:1787829691.238990:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00080000:3.0:1787829691.238995:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:1.0:1787829691.238998:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787829691.239004:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829691.239005:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 02000000:00000001:3.0:1787829691.239012:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1787829691.239013:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829691.239021:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829691.239023:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787829691.239030:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829691.239032:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829691.239037:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 02000000:00000001:1.0:1787829691.239041:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000008:00100000:3.0:1787829691.239047:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829691.239055:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c7016580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646855040:192.168.203.143@tcp:250:kworker.0 00000008:00000001:3.0:1787829691.239056:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829691.239063:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829691.239069:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1787829691.239072:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1787829691.239077:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1787829691.239078:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829691.239084:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829691.239085:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787829691.239091:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1787829691.239093:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1787829691.239096:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829691.239103:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829691.239107:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829691.239119:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c7015a40 x1874670646855168/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787829691.239125:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787829691.239131:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369545088 : -126234340006528 : ffff8d30c7016580) 00000100:00000001:3.0:1787829691.239135:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787829691.239140:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:3.0:1787829691.239142:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829691.239150:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787829691.239155:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829691.239160:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1787829691.239163:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787829691.239181:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787829691.239185:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787829691.239189:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4491 00000100:00080000:3.0:1787829691.239195:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787829691.239198:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829691.239200:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:3.0:1787829691.239204:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1787829691.239206:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:1.0:1787829691.239226:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829691.239226:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000020:00000001:3.0:1787829691.239235:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:1.0:1787829691.239238:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4299165440 00000100:00000001:3.0:1787829691.239240:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829691.239249:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:1.0:1787829691.239252:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 00000100:00080000:3.0:1787829691.239258:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:3.0:1787829691.239264:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829691.239267:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787829691.239271:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829691.239274:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4299165440 02000000:00000001:3.0:1787829691.239275:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829691.239283:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:1.0:1787829691.239287:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 00000008:00100000:3.0:1787829691.239289:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00000001:1.0:1787829691.239330:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000008:00000001:3.0:1787829691.239332:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829691.239336:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829691.239338:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829691.239345:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829691.239349:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:1.0:1787829691.239350:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:3.0:1787829691.239354:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829691.239359:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787829691.239361:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:1.0:1787829691.239362:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 02000000:00000001:3.0:1787829691.239364:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829691.239367:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:1.0:1787829691.239372:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1787829691.239373:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829691.239376:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1787829691.239380:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787829691.239384:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00100000:3.0:1787829691.239385:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c70161c0 x1874670646855296/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787829691.239387:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829691.239393:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787829691.239397:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829691.239404:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829691.239407:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787829691.239413:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829691.239414:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c7015a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646855168:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829691.239420:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829691.239423:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00080000:3.0:1787829691.239427:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 02000000:00000001:1.0:1787829691.239443:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787829691.239448:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787829691.239451:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 02000000:00000001:1.0:1787829691.239454:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:3.0:1787829691.239456:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787829691.239470:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 02000000:00000001:1.0:1787829691.239474:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829691.239476:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787829691.239484:0:468:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4491 00000100:00000001:1.0:1787829691.239488:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00080000:3.0:1787829691.239492:0:468:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1787829691.239493:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369542208 : -126234340009408 : ffff8d30c7015a40) 00000100:00000001:3.0:1787829691.239499:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787829691.239503:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:3.0:1787829691.239504:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:3.0:1787829691.239510:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1787829691.239512:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1787829691.239515:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000020:00000001:3.0:1787829691.239519:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:3.0:1787829691.239523:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829691.239526:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:3.0:1787829691.239532:0:468:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:3.0:1787829691.239537:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829691.239541:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829691.239542:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787829691.239548:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829691.239552:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4299164182 02000000:00000001:3.0:1787829691.239554:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829691.239560:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4299165440 00000100:00000001:3.0:1787829691.239560:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:3.0:1787829691.239565:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00000001:1.0:1787829691.239567:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4299164182 00000008:00000001:3.0:1787829691.239571:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829691.239574:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4299165440 00000100:00000001:3.0:1787829691.239575:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829691.239582:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829691.239586:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:1.0:1787829691.239587:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829691.239589:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829691.239593:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829691.239594:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787829691.239597:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787829691.239601:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829691.239606:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829691.239612:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c7014f00 x1874670646855424/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787829691.239644:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829691.239650:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787829691.239653:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:1.0:1787829691.239655:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787829691.239657:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1787829691.239661:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787829691.239665:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787829691.239668:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829691.239674:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 02000000:00000001:1.0:1787829691.239675:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787829691.239680:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829691.239682:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829691.239688:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829691.239691:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829691.239697:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c70161c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646855296:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787829691.239714:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787829691.239717:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787829691.239719:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829691.239722:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787829691.239726:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829691.239733:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787829691.239735:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369544128 : -126234340007488 : ffff8d30c70161c0) 00000100:00000001:1.0:1787829691.239739:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:1.0:1787829691.239760:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829691.239765:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4299164183 00000100:00000001:1.0:1787829691.239767:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4299165440 00000100:00000001:1.0:1787829691.239770:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4299164183 00000100:00000001:1.0:1787829691.239773:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4299165440 00000100:00000001:1.0:1787829691.239775:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829691.239778:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829691.239781:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829691.239783:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829691.239787:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787829691.239788:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:1.0:1787829691.239790:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787829691.239792:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1787829691.239793:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787829691.239795:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787829691.239797:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829691.239800:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787829691.239802:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829691.239805:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829691.239809:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c7014f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646855424:192.168.203.143@tcp:8:kworker.0 00000100:00000001:1.0:1787829691.239813:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787829691.239816:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787829691.239818:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829691.239820:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787829691.239824:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829691.239831:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787829691.239833:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369539328 : -126234340012288 : ffff8d30c7014f00) 00000100:00000001:1.0:1787829691.239838:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:1.0:1787829691.239850:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829691.239855:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4299164196 00000100:00000001:1.0:1787829691.239858:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4299165440 00000100:00000001:1.0:1787829691.239861:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4299164196 00000100:00000001:1.0:1787829691.239863:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4299165440 00000100:00000001:1.0:1787829691.239866:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829691.239869:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829691.239872:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829691.239874:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829691.239878:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787829691.239880:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:1.0:1787829691.239881:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787829691.239882:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1787829691.239884:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829691.239886:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829692.134694:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829692.134699:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829692.134773:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829692.134777:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829692.134781:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829693.158717:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829693.158723:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829693.158775:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829693.158778:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829693.158781:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829694.182681:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829694.182687:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829694.182741:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829694.182744:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829694.182747:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:1.0:1787829694.310939:0:2380:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.143:988 00000400:00000100:1.0:1787829694.310978:0:2380:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.143@tcp at host 192.168.203.143 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1787829694.310995:0:2380:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.143@tcp 00000100:00000001:1.0:1787829694.311008:0:2380:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787829694.311037:0:2380:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829694.311040:0:2380:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829694.311043:0:2380:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:1.0:1787829694.311054:0:2380:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.143@tcp 00000100:00000001:1.0:1787829694.311059:0:2380:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787829694.311063:0:2380:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829694.311064:0:2380:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829694.311066:0:2380:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:1.0:1787829694.311069:0:2380:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.143@tcp 00000100:00000001:1.0:1787829694.311073:0:2380:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787829694.311075:0:2380:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829694.311077:0:2380:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829694.311078:0:2380:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:1.0:1787829694.311082:0:2380:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.143@tcp 00000100:00000001:1.0:1787829694.311086:0:2380:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787829694.311087:0:2380:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829694.311089:0:2380:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829694.311090:0:2380:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1787829694.311152:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1787829694.311156:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787829694.311159:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1787829694.311170:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829691/real 1787829694] req@ffff8d30c7014f00 x1874670646855424/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829716 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1787829694.311186:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787829694.311188:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829694.311193:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829694.311196:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829694.311198:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829694.311200:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829694.311202:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829694.311208:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c7014f00 x1874670646855424/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829716 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829694.311222:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829694.311226:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829694.311229:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829694.311231:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829694.311232:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829694.311235:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829694.311238:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829694.311242:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829694.311246:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829694.311249:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829694.311254:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c7014f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646855424:192.168.203.143@tcp:8:kworker.0 00000100:00000001:3.0:1787829694.311259:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1787829694.311263:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829691/real 1787829694] req@ffff8d30c70161c0 x1874670646855296/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829716 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1787829694.311287:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787829694.311288:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829694.311291:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829694.311292:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829694.311294:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829694.311296:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829694.311297:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829694.311301:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c70161c0 x1874670646855296/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829716 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829694.311308:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829694.311310:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829694.311313:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829694.311314:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829694.311315:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829694.311317:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829694.311320:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829694.311322:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829694.311326:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829694.311328:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829694.311337:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c70161c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646855296:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829694.311341:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1787829694.311344:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829691/real 1787829694] req@ffff8d30c7015a40 x1874670646855168/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829716 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1787829694.311352:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787829694.311354:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829694.311357:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829694.311359:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829694.311360:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829694.311362:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829694.311363:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829694.311368:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c7015a40 x1874670646855168/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829716 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829694.311374:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829694.311376:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829694.311378:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829694.311380:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829694.311381:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829694.311384:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829694.311385:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829694.311388:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829694.311391:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829694.311394:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829694.311397:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c7015a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646855168:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829694.311402:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1787829694.311406:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829691/real 1787829694] req@ffff8d30c7016580 x1874670646855040/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829716 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1787829694.311414:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787829694.311415:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829694.311419:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829694.311420:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829694.311422:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829694.311424:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829694.311425:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829694.311430:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c7016580 x1874670646855040/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829716 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829694.311441:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829694.311443:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829694.311446:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829694.311447:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829694.311449:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829694.311451:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829694.311452:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829694.311455:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829694.311458:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829694.311461:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829694.311464:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c7016580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646855040:192.168.203.143@tcp:250:kworker.0 00000100:00000001:3.0:1787829694.311468:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829694.311471:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829694.311473:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829694.311476:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829694.311480:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829694.311484:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829694.311486:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829694.311491:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829694.311493:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829694.311497:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829694.311498:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829694.311500:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829694.311502:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829694.311503:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829694.311505:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829694.311506:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829694.311508:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829694.311510:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829694.311511:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829694.311513:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829694.311515:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829694.311517:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829694.311518:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829694.311519:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829694.311522:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829694.311524:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829694.311525:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829694.311527:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829694.311529:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829694.311530:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829694.311532:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829694.311534:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829694.311535:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829694.311536:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829694.311538:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829694.311539:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829694.311541:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829694.311542:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829694.311544:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829694.311546:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829694.311547:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829694.311550:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829694.311557:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787829694.311559:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829694.311562:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1787829694.311563:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829695.206710:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829695.206716:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829695.206773:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829695.206776:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829695.206779:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829696.230668:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829696.230674:0:815:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1787829696.230676:0:815:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1787829696.230679:0:815:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829696.230682:0:815:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1787829696.230685:0:815:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829696.230686:0:815:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829696.230688:0:815:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829696.230733:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829696.230736:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829696.230739:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787829696.358698:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829696.358711:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1787829696.358713:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787829696.358717:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787829696.358720:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787829696.358727:0:468:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 4496 00000100:00080000:3.0:1787829696.358736:0:468:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787829696.358738:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829696.358740:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:3.0:1787829696.358742:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1787829696.358745:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1787829696.358747:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:3.0:1787829696.358749:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:3.0:1787829696.358750:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829696.358751:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:3.0:1787829696.358755:0:468:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:3.0:1787829696.358757:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829696.358759:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787829696.358762:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829696.358764:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829696.358765:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:3.0:1787829696.358767:0:468:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829696.358774:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787829696.358775:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829696.358778:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787829696.358780:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787829696.358783:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829696.358791:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829696.358799:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c7016580 x1874670646855552/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787829696.358807:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787829696.358811:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829696.358827:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787829696.358829:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829696.358832:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1787829696.358833:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787829696.358834:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787829696.358836:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787829696.358838:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4496 00000100:00080000:3.0:1787829696.358841:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787829696.358842:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829696.358843:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:3.0:1787829696.358845:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1787829696.358846:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1787829696.358848:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000020:00000001:3.0:1787829696.358851:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:3.0:1787829696.358853:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829696.358855:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:3.0:1787829696.358859:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:1.0:1787829696.358866:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1787829696.358871:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829696.358872:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1787829696.358876:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 02000000:00000001:3.0:1787829696.358877:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1787829696.358879:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787829696.358881:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829696.358884:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829696.358885:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787829696.358888:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829696.358890:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829696.358892:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829696.358896:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00100000:1.0:1787829696.358899:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c7016580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646855552:192.168.203.143@tcp:250:kworker.0 00000008:00100000:3.0:1787829696.358903:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00000001:1.0:1787829696.358904:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787829696.358910:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000008:00000001:3.0:1787829696.358910:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829696.358915:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829696.358916:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829696.358922:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1787829696.358925:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787829696.358928:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829696.358930:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829696.358931:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787829696.358935:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787829696.358937:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829696.358944:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829696.358949:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00100000:3.0:1787829696.358951:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c7015a40 x1874670646855680/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787829696.358953:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369545088 : -126234340006528 : ffff8d30c7016580) 00000100:00000001:1.0:1787829696.358960:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:3.0:1787829696.358967:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787829696.358969:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829696.358979:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787829696.358982:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829696.358985:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1787829696.358987:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787829696.358989:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787829696.358992:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787829696.358994:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4496 00000100:00080000:3.0:1787829696.358999:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787829696.359001:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829696.359003:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:3.0:1787829696.359005:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1787829696.359007:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1787829696.359009:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000020:00000001:3.0:1787829696.359011:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:3.0:1787829696.359015:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000800:00000100:1.0:1787829696.359017:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:3.0:1787829696.359018:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:3.0:1787829696.359026:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:3.0:1787829696.359030:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829696.359032:0:2390:0:(events.c:34:request_out_callback()) Process entered 02000000:00000001:3.0:1787829696.359033:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1787829696.359038:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 02000000:00000001:3.0:1787829696.359039:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829696.359043:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829696.359044:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829696.359050:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1787829696.359051:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:3.0:1787829696.359055:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00000001:1.0:1787829696.359058:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:3.0:1787829696.359059:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829696.359064:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4299170560 00000100:00000001:3.0:1787829696.359064:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829696.359069:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787829696.359072:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829696.359074:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787829696.359076:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:1.0:1787829696.359077:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 02000000:00000001:3.0:1787829696.359079:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829696.359088:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829696.359093:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c70161c0 x1874670646855808/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787829696.359101:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787829696.359104:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4299170560 00000100:00000001:3.0:1787829696.359104:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829696.359107:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787829696.359112:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1787829696.359113:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 00000100:00080000:3.0:1787829696.359118:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1787829696.359120:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829696.359122:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:1.0:1787829696.359126:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787829696.359127:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787829696.359133:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00000001:3.0:1787829696.359134:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787829696.359140:0:468:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4496 00000100:00100000:1.0:1787829696.359146:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829696/real 1787829696] req@ffff8d30c7016580 x1874670646855552/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829721 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787829696.359147:0:468:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787829696.359152:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829696.359154:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:3.0:1787829696.359161:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1787829696.359164:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000400:00000001:1.0:1787829696.359165:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000020:00000001:3.0:1787829696.359166:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000400:00000001:1.0:1787829696.359169:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000020:00000001:3.0:1787829696.359172:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:1.0:1787829696.359175:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829696.359176:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787829696.359180:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829696.359181:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:1.0:1787829696.359184:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:3.0:1787829696.359189:0:468:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:1.0:1787829696.359190:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829696.359194:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829696.359195:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829696.359200:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00100000:1.0:1787829696.359203:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c7016580 x1874670646855552/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829721 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1787829696.359205:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829696.359214:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829696.359219:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787829696.359224:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:3.0:1787829696.359225:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829696.359229:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829696.359231:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:1.0:1787829696.359233:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000008:00100000:3.0:1787829696.359235:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00000001:1.0:1787829696.359238:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000008:00000001:3.0:1787829696.359242:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829696.359243:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829696.359245:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1787829696.359249:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 02000000:00000001:3.0:1787829696.359252:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787829696.359269:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1787829696.359272:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829696.359274:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787829696.359279:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:1.0:1787829696.359280:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 02000000:00000001:3.0:1787829696.359282:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829696.359286:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c7016580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646855552:192.168.203.143@tcp:250:kworker.0 00000100:00000001:3.0:1787829696.359290:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829696.359294:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829696.359299:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00100000:3.0:1787829696.359300:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c7014f00 x1874670646855936/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787829696.359302:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787829696.359307:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787829696.359313:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00000001:3.0:1787829696.359315:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000020:00000001:1.0:1787829696.359316:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000100:00000001:3.0:1787829696.359320:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829696.359321:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 00000100:00000001:3.0:1787829696.359327:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829696.359328:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787829696.359331:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787829696.359335:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787829696.359337:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829696.359340:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829696.359348:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787829696.359350:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829696.359354:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787829696.359355:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1787829696.359357:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787829696.359359:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787829696.359361:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829696.359364:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787829696.359367:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829696.359369:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829696.359374:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c7015a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646855680:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787829696.359378:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787829696.359380:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787829696.359381:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829696.359383:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787829696.359386:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829696.359392:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787829696.359394:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369542208 : -126234340009408 : ffff8d30c7015a40) 00000100:00000001:1.0:1787829696.359399:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1787829696.359422:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787829696.359427:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787829696.359429:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829696.359432:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829696.359434:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787829696.359436:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829696.359441:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4299169302 00000100:00000001:1.0:1787829696.359444:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4299170560 00000100:00000001:1.0:1787829696.359447:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4299169302 00000100:00000001:1.0:1787829696.359449:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4299170560 00000100:00000001:1.0:1787829696.359451:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829696.359453:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829696.359455:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787829696.359459:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829696/real 1787829696] req@ffff8d30c7015a40 x1874670646855680/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829721 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787829696.359468:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787829696.359469:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787829696.359478:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787829696.359479:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787829696.359480:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829696.359482:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829696.359484:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787829696.359488:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c7015a40 x1874670646855680/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829721 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787829696.359494:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829696.359497:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787829696.359500:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829696.359501:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829696.359503:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829696.359505:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787829696.359507:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787829696.359509:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829696.359513:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787829696.359516:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787829696.359519:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c7015a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646855680:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787829696.359523:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787829696.359525:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787829696.359526:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829696.359528:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787829696.359530:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829696.359533:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829696.359535:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c70161c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646855808:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787829696.359544:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787829696.359546:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787829696.359547:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829696.359549:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787829696.359552:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829696.359557:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787829696.359558:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369544128 : -126234340007488 : ffff8d30c70161c0) 00000100:00000001:1.0:1787829696.359562:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1787829696.359573:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787829696.359578:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787829696.359580:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829696.359582:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829696.359584:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787829696.359587:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829696.359591:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4299169302 00000100:00000001:1.0:1787829696.359593:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4299170560 00000100:00000001:1.0:1787829696.359595:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4299169302 00000100:00000001:1.0:1787829696.359596:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4299170560 00000100:00000001:1.0:1787829696.359598:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829696.359621:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829696.359623:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787829696.359627:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829696/real 1787829696] req@ffff8d30c70161c0 x1874670646855808/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829721 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787829696.359638:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787829696.359639:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787829696.359641:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787829696.359642:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787829696.359643:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829696.359645:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829696.359647:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787829696.359651:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c70161c0 x1874670646855808/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829721 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787829696.359659:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829696.359661:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787829696.359663:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829696.359665:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829696.359667:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829696.359668:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787829696.359670:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787829696.359673:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829696.359676:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787829696.359678:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787829696.359682:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c70161c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646855808:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787829696.359685:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787829696.359686:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787829696.359688:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829696.359690:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787829696.359692:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829696.359694:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829696.359697:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c7014f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646855936:192.168.203.143@tcp:8:kworker.0 00000100:00000001:1.0:1787829696.359700:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787829696.359701:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787829696.359703:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829696.359705:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787829696.359708:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829696.359715:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787829696.359717:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369539328 : -126234340012288 : ffff8d30c7014f00) 00000100:00000001:1.0:1787829696.359721:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1787829696.359735:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787829696.359739:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787829696.359742:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829696.359743:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829696.359745:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787829696.359747:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829696.359750:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4299169316 00000100:00000001:1.0:1787829696.359753:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4299170560 00000100:00000001:1.0:1787829696.359756:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4299169316 00000100:00000001:1.0:1787829696.359758:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4299170560 00000100:00000001:1.0:1787829696.359761:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829696.359762:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829696.359764:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787829696.359768:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829696/real 1787829696] req@ffff8d30c7014f00 x1874670646855936/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829721 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787829696.359776:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787829696.359777:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787829696.359780:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787829696.359781:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787829696.359783:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829696.359785:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829696.359786:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787829696.359790:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c7014f00 x1874670646855936/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829721 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787829696.359796:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829696.359802:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787829696.359806:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829696.359808:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829696.359809:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829696.359810:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787829696.359812:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787829696.359814:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829696.359817:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787829696.359820:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787829696.359823:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c7014f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646855936:192.168.203.143@tcp:8:kworker.0 00000100:00000001:1.0:1787829696.359827:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829696.359829:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829696.359830:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787829696.359832:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787829696.359835:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787829696.359837:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787829696.359839:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787829696.359842:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787829696.359843:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787829696.359845:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787829696.359846:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829696.359847:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829696.359848:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787829696.359849:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787829696.359851:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787829696.359852:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787829696.359853:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787829696.359855:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787829696.359856:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787829696.359858:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787829696.359859:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829696.359860:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829696.359861:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787829696.359862:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787829696.359864:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787829696.359864:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787829696.359866:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787829696.359867:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787829696.359868:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787829696.359870:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787829696.359876:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829696.359878:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829696.359882:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787829696.359883:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829696.359885:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787829696.359887:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829697.254703:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829697.254710:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829697.254781:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829697.254784:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829697.254788:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829698.278725:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829698.278730:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829698.278775:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829698.278779:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829698.278781:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829699.302702:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829699.302707:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829699.302746:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829699.302749:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829699.302752:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829700.326716:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829700.326722:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829700.326781:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829700.326785:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829700.326788:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829701.350706:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829701.350713:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829701.350764:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829701.350767:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829701.350770:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787829701.478723:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829701.478730:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1787829701.478733:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787829701.478736:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787829701.478739:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787829701.478745:0:468:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 4501 00000100:00080000:3.0:1787829701.478755:0:468:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787829701.478757:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829701.478759:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:3.0:1787829701.478762:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1787829701.478764:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1787829701.478767:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:3.0:1787829701.478769:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:3.0:1787829701.478771:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829701.478772:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:3.0:1787829701.478776:0:468:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:3.0:1787829701.478777:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829701.478780:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787829701.478783:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829701.478785:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829701.478787:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:3.0:1787829701.478790:0:468:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829701.478802:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787829701.478804:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829701.478808:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787829701.478810:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787829701.478815:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829701.478825:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829701.478836:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c7014b40 x1874670646856064/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787829701.478846:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787829701.478850:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829701.478867:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787829701.478870:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829701.478874:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1787829701.478876:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787829701.478878:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787829701.478881:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787829701.478884:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4501 00000100:00080000:3.0:1787829701.478888:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787829701.478890:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829701.478891:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:3.0:1787829701.478894:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1787829701.478895:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1787829701.478896:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000020:00000001:3.0:1787829701.478898:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:3.0:1787829701.478899:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829701.478901:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:3.0:1787829701.478903:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:3.0:1787829701.478905:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829701.478906:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787829701.478911:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829701.478912:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829701.478914:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:3.0:1787829701.478918:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1787829701.478920:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829701.478922:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829701.478926:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787829701.478927:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829701.478929:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787829701.478930:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787829701.478931:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829701.478934:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1787829701.478935:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829701.478940:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00100000:3.0:1787829701.478942:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c7015680 x1874670646856192/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787829701.478943:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787829701.478947:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787829701.478950:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829701.478956:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:3.0:1787829701.478956:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787829701.478959:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829701.478961:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829701.478963:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829701.478965:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787829701.478967:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1787829701.478973:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c7014b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646856064:192.168.203.143@tcp:250:kworker.0 00000100:00080000:3.0:1787829701.478973:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1787829701.478976:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787829701.478979:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787829701.478981:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:3.0:1787829701.478983:0:468:0:(import.c:485:import_select_connection()) Process entered 02000000:00000001:1.0:1787829701.478986:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000100:00080000:3.0:1787829701.478987:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4501 02000000:00000001:1.0:1787829701.478991:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787829701.478992:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 02000000:00000001:1.0:1787829701.478994:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:3.0:1787829701.478995:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829701.478998:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:3.0:1787829701.478999:0:468:0:(genops.c:792:class_conn2export()) Process entered 02000000:00000001:1.0:1787829701.479000:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829701.479001:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1787829701.479003:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000020:00000001:3.0:1787829701.479006:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:3.0:1787829701.479008:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829701.479010:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:3.0:1787829701.479012:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:3.0:1787829701.479013:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829701.479014:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787829701.479016:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829701.479017:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 02000000:00000001:3.0:1787829701.479018:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829701.479021:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369538368 : -126234340013248 : ffff8d30c7014b40) 00000100:00000001:3.0:1787829701.479022:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:3.0:1787829701.479024:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00000001:1.0:1787829701.479027:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000008:00000001:3.0:1787829701.479027:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829701.479030:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829701.479034:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787829701.479035:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829701.479036:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787829701.479037:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787829701.479039:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829701.479041:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829701.479045:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c70152c0 x1874670646856320/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787829701.479050:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787829701.479052:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829701.479053:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787829701.479055:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829701.479057:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1787829701.479058:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787829701.479059:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787829701.479061:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787829701.479063:0:468:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4501 00000100:00080000:3.0:1787829701.479065:0:468:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787829701.479066:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829701.479067:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:3.0:1787829701.479069:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1787829701.479072:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000800:00000100:1.0:1787829701.479073:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000020:00000001:3.0:1787829701.479074:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000020:00000001:3.0:1787829701.479078:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:3.0:1787829701.479079:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829701.479081:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:3.0:1787829701.479084:0:468:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:3.0:1787829701.479086:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829701.479087:0:2390:0:(events.c:34:request_out_callback()) Process entered 02000000:00000001:3.0:1787829701.479088:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787829701.479092:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829701.479093:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 02000000:00000001:3.0:1787829701.479094:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829701.479097:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:1.0:1787829701.479098:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000008:00100000:3.0:1787829701.479100:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00000001:1.0:1787829701.479101:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000008:00000001:3.0:1787829701.479105:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829701.479107:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829701.479108:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829701.479110:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787829701.479113:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829701.479114:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4299175680 00000100:00000001:3.0:1787829701.479114:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787829701.479117:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787829701.479119:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829701.479123:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829701.479125:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 00000100:00100000:3.0:1787829701.479128:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c70143c0 x1874670646856448/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787829701.479134:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787829701.479136:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829701.479138:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829701.479139:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4299175680 00000100:00000001:1.0:1787829701.479144:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 00000100:00000001:1.0:1787829701.479148:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829701.479150:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829701.479152:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787829701.479159:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829701/real 1787829701] req@ffff8d30c7014b40 x1874670646856064/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829726 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787829701.479168:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787829701.479171:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787829701.479176:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787829701.479179:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787829701.479182:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829701.479186:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829701.479188:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787829701.479194:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c7014b40 x1874670646856064/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829726 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787829701.479203:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829701.479206:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787829701.479209:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829701.479211:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829701.479212:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829701.479215:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787829701.479217:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787829701.479232:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829701.479237:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787829701.479240:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787829701.479243:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c7014b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646856064:192.168.203.143@tcp:250:kworker.0 00000100:00000001:1.0:1787829701.479249:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829701.479251:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829701.479253:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787829701.479255:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787829701.479258:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787829701.479261:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787829701.479273:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787829701.479277:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787829701.479279:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787829701.479281:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787829701.479282:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829701.479285:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829701.479291:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787829701.479293:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829701.479295:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787829701.479296:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1787829701.479297:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787829701.479298:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787829701.479299:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829701.479301:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787829701.479302:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829701.479303:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829701.479306:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c7015680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646856192:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787829701.479308:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787829701.479310:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787829701.479311:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829701.479312:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787829701.479314:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829701.479319:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787829701.479320:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369541248 : -126234340010368 : ffff8d30c7015680) 00000100:00000001:1.0:1787829701.479323:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1787829701.479333:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787829701.479336:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787829701.479338:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829701.479339:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829701.479340:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787829701.479342:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829701.479346:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4299174422 00000100:00000001:1.0:1787829701.479359:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4299175680 00000100:00000001:1.0:1787829701.479360:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4299174422 00000100:00000001:1.0:1787829701.479362:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4299175680 00000100:00000001:1.0:1787829701.479363:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829701.479365:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829701.479366:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787829701.479368:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829701/real 1787829701] req@ffff8d30c7015680 x1874670646856192/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829726 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787829701.479373:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787829701.479374:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787829701.479376:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787829701.479377:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787829701.479378:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829701.479379:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829701.479380:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787829701.479382:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c7015680 x1874670646856192/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829726 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787829701.479386:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829701.479388:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787829701.479389:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829701.479390:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829701.479391:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829701.479392:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787829701.479393:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787829701.479394:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829701.479396:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787829701.479398:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787829701.479399:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c7015680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646856192:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787829701.479404:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787829701.479405:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787829701.479406:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829701.479407:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787829701.479408:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829701.479409:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829701.479411:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c70152c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646856320:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787829701.479413:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787829701.479414:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787829701.479415:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829701.479416:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787829701.479417:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829701.479420:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787829701.479421:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369540288 : -126234340011328 : ffff8d30c70152c0) 00000100:00000001:1.0:1787829701.479423:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1787829701.479429:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787829701.479431:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787829701.479432:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829701.479433:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829701.479434:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787829701.479436:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829701.479438:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4299174422 00000100:00000001:1.0:1787829701.479439:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4299175680 00000100:00000001:1.0:1787829701.479441:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4299174422 00000100:00000001:1.0:1787829701.479442:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4299175680 00000100:00000001:1.0:1787829701.479444:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829701.479445:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829701.479446:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787829701.479448:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829701/real 1787829701] req@ffff8d30c70152c0 x1874670646856320/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829726 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787829701.479453:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787829701.479454:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787829701.479455:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787829701.479456:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787829701.479457:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829701.479458:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829701.479459:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787829701.479461:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c70152c0 x1874670646856320/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829726 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787829701.479465:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829701.479466:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787829701.479468:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829701.479469:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829701.479469:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829701.479470:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787829701.479471:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787829701.479473:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829701.479474:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787829701.479475:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787829701.479477:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c70152c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646856320:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787829701.479479:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787829701.479480:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787829701.479481:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829701.479482:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787829701.479483:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829701.479484:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829701.479486:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c70143c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646856448:192.168.203.143@tcp:8:kworker.0 00000100:00000001:1.0:1787829701.479488:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787829701.479489:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787829701.479490:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829701.479491:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787829701.479492:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829701.479496:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787829701.479497:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369536448 : -126234340015168 : ffff8d30c70143c0) 00000100:00000001:1.0:1787829701.479499:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1787829701.479505:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787829701.479507:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787829701.479508:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829701.479509:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829701.479510:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787829701.479512:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829701.479513:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4299174435 00000100:00000001:1.0:1787829701.479515:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4299175680 00000100:00000001:1.0:1787829701.479517:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4299174435 00000100:00000001:1.0:1787829701.479518:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4299175680 00000100:00000001:1.0:1787829701.479519:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829701.479521:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829701.479522:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787829701.479524:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829701/real 1787829701] req@ffff8d30c70143c0 x1874670646856448/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829726 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787829701.479528:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787829701.479529:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787829701.479530:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787829701.479531:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787829701.479532:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829701.479533:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829701.479534:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787829701.479536:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c70143c0 x1874670646856448/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829726 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787829701.479540:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829701.479541:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787829701.479543:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829701.479544:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829701.479545:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829701.479546:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787829701.479547:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787829701.479549:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829701.479550:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787829701.479552:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787829701.479553:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c70143c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646856448:192.168.203.143@tcp:8:kworker.0 00000100:00000001:1.0:1787829701.479556:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829701.479557:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829701.479557:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787829701.479558:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787829701.479560:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787829701.479560:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787829701.479561:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787829701.479563:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787829701.479563:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787829701.479564:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787829701.479565:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829701.479566:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829701.479567:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787829701.479568:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787829701.479569:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787829701.479569:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787829701.479570:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787829701.479571:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787829701.479571:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787829701.479572:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787829701.479573:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829701.479574:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829701.479575:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787829701.479576:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787829701.479577:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787829701.479578:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787829701.479579:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787829701.479579:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787829701.479580:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787829701.479581:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787829701.479582:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829701.479583:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829701.479585:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787829701.479586:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829701.479587:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787829701.479589:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829702.374730:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829702.374745:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829702.374831:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829702.374838:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829702.374842:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829703.398711:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829703.398718:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829703.398779:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829703.398784:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829703.398787:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829704.422737:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829704.422744:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829704.422846:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829704.422850:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829704.422853:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829705.446648:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829705.446655:0:815:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1787829705.446657:0:815:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1787829705.446659:0:815:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829705.446663:0:815:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1787829705.446665:0:815:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829705.446667:0:815:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829705.446669:0:815:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829705.446733:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829705.446737:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829705.446741:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829706.470690:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829706.470695:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829706.470753:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829706.470756:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829706.470760:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787829706.598699:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829706.598710:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1787829706.598712:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1787829706.598716:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787829706.598720:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1787829706.598727:0:468:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 4507 00000100:00080000:1.0:1787829706.598738:0:468:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1787829706.598741:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787829706.598743:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:1.0:1787829706.598747:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:1.0:1787829706.598750:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:1.0:1787829706.598753:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:1.0:1787829706.598756:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:1.0:1787829706.598758:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787829706.598760:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:1.0:1787829706.598764:0:468:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:1.0:1787829706.598767:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829706.598769:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1787829706.598773:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829706.598776:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829706.598779:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:1.0:1787829706.598782:0:468:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829706.598793:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1787829706.598795:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829706.598798:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1787829706.598801:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1787829706.598805:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829706.598816:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829706.598825:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d13ef480 x1874670646856576/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787829706.598835:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787829706.598840:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829706.598855:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1787829706.598859:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829706.598862:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1787829706.598864:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1787829706.598866:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787829706.598870:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1787829706.598873:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4507 00000100:00080000:1.0:1787829706.598877:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787829706.598879:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787829706.598880:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787829706.598885:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:3.0:1787829706.598886:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000020:00000001:1.0:1787829706.598889:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:3.0:1787829706.598890:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000020:00000001:1.0:1787829706.598894:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:3.0:1787829706.598894:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000020:00000001:1.0:1787829706.598899:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:3.0:1787829706.598899:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829706.598906:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 02000000:00000001:3.0:1787829706.598908:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:1.0:1787829706.598910:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787829706.598913:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 02000000:00000001:3.0:1787829706.598915:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00080000:1.0:1787829706.598919:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 02000000:00000001:3.0:1787829706.598922:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829706.598924:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829706.598928:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00100000:3.0:1787829706.598933:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d13ef480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646856576:192.168.203.143@tcp:250:kworker.0 02000000:00000001:1.0:1787829706.598935:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829706.598942:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829706.598944:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:1.0:1787829706.598945:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 02000000:00000001:3.0:1787829706.598950:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000008:00100000:1.0:1787829706.598952:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 02000000:00000001:3.0:1787829706.598955:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:1.0:1787829706.598957:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829706.598960:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:1.0:1787829706.598962:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829706.598965:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829706.598968:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1787829706.598972:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829706.598974:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1787829706.598976:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1787829706.598978:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829706.598981:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787829706.598984:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839541351552 : -126234168200064 : ffff8d30d13ef480) 00000100:00000001:1.0:1787829706.598985:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829706.598988:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00100000:1.0:1787829706.598994:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d13ee580 x1874670646856704/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787829706.599003:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787829706.599006:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829706.599010:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1787829706.599014:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829706.599017:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:1.0:1787829706.599019:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1787829706.599021:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:3.0:1787829706.599026:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787829706.599027:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1787829706.599033:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4507 00000100:00000001:3.0:1787829706.599039:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00080000:1.0:1787829706.599044:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787829706.599044:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829706.599046:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829706.599049:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829706.599050:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787829706.599053:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:3.0:1787829706.599057:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829706.599058:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:1.0:1787829706.599062:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:3.0:1787829706.599062:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4299180800 00000020:00000001:1.0:1787829706.599067:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000020:00000001:1.0:1787829706.599070:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:1.0:1787829706.599073:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829706.599074:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 00000100:00000001:1.0:1787829706.599075:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:1.0:1787829706.599081:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:3.0:1787829706.599082:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4299180800 00000100:00000001:1.0:1787829706.599088:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829706.599089:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 02000000:00000001:1.0:1787829706.599091:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:3.0:1787829706.599093:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829706.599096:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829706.599097:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829706.599103:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829706.599103:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00000001:1.0:1787829706.599108:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00100000:3.0:1787829706.599111:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829706/real 1787829706] req@ffff8d30d13ef480 x1874670646856576/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829731 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:1.0:1787829706.599112:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1787829706.599119:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829706.599123:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:3.0:1787829706.599130:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 02000000:00000001:1.0:1787829706.599132:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000400:00000001:3.0:1787829706.599133:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 02000000:00000001:1.0:1787829706.599137:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829706.599139:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787829706.599141:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:3.0:1787829706.599143:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 02000000:00000001:1.0:1787829706.599144:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:3.0:1787829706.599145:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:1.0:1787829706.599148:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829706.599148:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829706.599151:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829706.599156:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829706.599156:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d13ef480 x1874670646856576/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829731 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829706.599167:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787829706.599168:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d13ee1c0 x1874670646856832/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787829706.599170:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829706.599175:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829706.599180:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829706.599182:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829706.599198:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829706.599200:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787829706.599206:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00080000:3.0:1787829706.599206:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00000001:1.0:1787829706.599212:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787829706.599214:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787829706.599218:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829706.599218:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00080000:1.0:1787829706.599224:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1787829706.599225:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d13ef480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646856576:192.168.203.143@tcp:250:kworker.0 00000100:00080000:1.0:1787829706.599234:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1787829706.599236:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829706.599237:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:3.0:1787829706.599239:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00080000:1.0:1787829706.599241:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787829706.599242:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829706.599246:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 00000100:00000001:1.0:1787829706.599247:0:468:0:(import.c:485:import_select_connection()) Process entered 02000000:00000001:3.0:1787829706.599250:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00080000:1.0:1787829706.599254:0:468:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4507 00000020:00000001:3.0:1787829706.599255:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829706.599258:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 00000100:00080000:1.0:1787829706.599262:0:468:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 02000000:00000001:3.0:1787829706.599263:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829706.599267:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787829706.599268:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787829706.599272:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:3.0:1787829706.599273:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000020:00000001:1.0:1787829706.599275:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:3.0:1787829706.599277:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:1.0:1787829706.599278:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:1.0:1787829706.599282:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:3.0:1787829706.599284:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:1.0:1787829706.599288:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:1.0:1787829706.599292:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787829706.599293:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:3.0:1787829706.599295:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00080000:1.0:1787829706.599296:0:468:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:3.0:1787829706.599300:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829706.599302:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829706.599306:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:3.0:1787829706.599306:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1787829706.599311:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 02000000:00000001:1.0:1787829706.599312:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829706.599317:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829706.599317:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787829706.599320:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787829706.599322:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:3.0:1787829706.599323:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000008:00100000:1.0:1787829706.599326:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 02000000:00000001:3.0:1787829706.599327:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787829706.599330:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000008:00000001:1.0:1787829706.599333:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829706.599334:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829706.599339:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829706.599341:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d13ee580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646856704:192.168.203.143@tcp:38:kworker.0 02000000:00000001:1.0:1787829706.599346:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:3.0:1787829706.599346:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1787829706.599348:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787829706.599351:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829706.599351:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829706.599357:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787829706.599357:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787829706.599361:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787829706.599363:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829706.599365:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829706.599372:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829706.599373:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787829706.599377:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839541347712 : -126234168203904 : ffff8d30d13ee580) 00000100:00000001:3.0:1787829706.599381:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00100000:1.0:1787829706.599382:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d13eda40 x1874670646856960/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787829706.599392:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000800:00000100:3.0:1787829706.599396:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787829706.599397:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829706.599402:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787829706.599403:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829706.599406:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829706.599409:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829706.599410:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1787829706.599414:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829706.599417:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4299179542 00000100:00000001:3.0:1787829706.599420:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4299180800 00000100:00000001:3.0:1787829706.599422:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4299179542 00000100:00000001:3.0:1787829706.599424:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4299180800 00000100:00000001:3.0:1787829706.599426:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829706.599428:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829706.599429:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1787829706.599433:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829706/real 1787829706] req@ffff8d30d13ee580 x1874670646856704/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829731 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1787829706.599439:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787829706.599440:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829706.599442:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829706.599443:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829706.599444:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829706.599445:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829706.599447:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829706.599449:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d13ee580 x1874670646856704/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829731 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829706.599454:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829706.599456:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829706.599457:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829706.599458:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829706.599459:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829706.599460:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829706.599461:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829706.599463:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829706.599465:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829706.599466:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829706.599468:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d13ee580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646856704:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829706.599471:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787829706.599472:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787829706.599473:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829706.599474:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787829706.599475:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829706.599477:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829706.599478:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d13ee1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646856832:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829706.599480:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1787829706.599481:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1787829706.599482:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829706.599484:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1787829706.599486:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829706.599488:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787829706.599489:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839541346752 : -126234168204864 : ffff8d30d13ee1c0) 00000100:00000001:3.0:1787829706.599491:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1787829706.599498:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:3.0:1787829706.599501:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787829706.599502:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829706.599503:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829706.599505:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1787829706.599506:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829706.599508:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4299179542 00000100:00000001:3.0:1787829706.599510:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4299180800 00000100:00000001:3.0:1787829706.599511:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4299179542 00000100:00000001:3.0:1787829706.599513:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4299180800 00000100:00000001:3.0:1787829706.599514:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829706.599515:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829706.599516:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1787829706.599519:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829706/real 1787829706] req@ffff8d30d13ee1c0 x1874670646856832/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829731 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1787829706.599524:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787829706.599524:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829706.599526:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829706.599527:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829706.599528:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829706.599529:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829706.599529:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829706.599532:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d13ee1c0 x1874670646856832/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829731 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829706.599536:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829706.599537:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829706.599538:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829706.599539:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829706.599540:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829706.599541:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829706.599542:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829706.599543:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829706.599546:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829706.599547:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829706.599549:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d13ee1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646856832:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829706.599552:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829706.599553:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829706.599554:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829706.599555:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829706.599556:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829706.599557:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829706.599558:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829706.599559:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829706.599560:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829706.599562:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829706.599563:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829706.599564:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829706.599564:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829706.599565:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829706.599566:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829706.599567:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829706.599568:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829706.599569:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829706.599570:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829706.599571:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829706.599572:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829706.599573:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829706.599576:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787829706.599576:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829706.599578:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1787829706.599579:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787829706.599579:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787829706.599580:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787829706.599581:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829706.599583:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787829706.599584:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829706.599585:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829706.599587:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d13eda40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646856960:192.168.203.143@tcp:8:kworker.0 00000100:00000001:3.0:1787829706.599589:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1787829706.599590:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1787829706.599591:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829706.599593:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1787829706.599594:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829706.599598:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787829706.599599:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839541344832 : -126234168206784 : ffff8d30d13eda40) 00000100:00000001:3.0:1787829706.599619:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1787829706.599628:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:3.0:1787829706.599631:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787829706.599632:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829706.599633:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829706.599635:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1787829706.599636:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829706.599638:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4299179556 00000100:00000001:3.0:1787829706.599640:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4299180800 00000100:00000001:3.0:1787829706.599642:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4299179556 00000100:00000001:3.0:1787829706.599644:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4299180800 00000100:00000001:3.0:1787829706.599647:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829706.599649:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829706.599651:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1787829706.599656:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829706/real 1787829706] req@ffff8d30d13eda40 x1874670646856960/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829731 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1787829706.599664:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787829706.599666:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829706.599669:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829706.599670:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829706.599672:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829706.599674:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829706.599675:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829706.599679:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d13eda40 x1874670646856960/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829731 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829706.599687:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829706.599690:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829706.599693:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829706.599694:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829706.599695:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829706.599697:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829706.599699:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829706.599700:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829706.599703:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829706.599704:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829706.599706:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d13eda40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646856960:192.168.203.143@tcp:8:kworker.0 00000100:00000001:3.0:1787829706.599709:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829706.599710:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829706.599711:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829706.599712:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829706.599714:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829706.599715:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829706.599716:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829706.599717:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829706.599718:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829706.599720:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829706.599720:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829706.599722:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829706.599724:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787829706.599725:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829706.599726:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1787829706.599727:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829707.494708:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829707.494713:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829707.494780:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829707.494785:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829707.494789:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829708.518691:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829708.518697:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829708.518777:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829708.518781:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829708.518785:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829709.542737:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829709.542743:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829709.542810:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829709.542815:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829709.542819:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829710.566768:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829710.566775:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829710.566847:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829710.566851:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829710.566855:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829711.590676:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829711.590682:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829711.590737:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829711.590741:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829711.590745:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787829711.718732:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829711.718741:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1787829711.718743:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787829711.718746:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787829711.718749:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787829711.718755:0:40798:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 4512 00000100:00080000:2.0:1787829711.718764:0:40798:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787829711.718766:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829711.718768:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787829711.718771:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787829711.718773:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787829711.718775:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:2.0:1787829711.718777:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:2.0:1787829711.718779:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829711.718780:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:2.0:1787829711.718783:0:40798:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787829711.718785:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829711.718787:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787829711.718790:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787829711.718792:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829711.718793:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:2.0:1787829711.718795:0:40798:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829711.718812:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787829711.718813:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829711.718816:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787829711.718817:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787829711.718821:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829711.718829:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787829711.718836:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2c3c0 x1874670646857088/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787829711.718844:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787829711.718848:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829711.718859:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787829711.718861:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829711.718863:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1787829711.718864:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787829711.718865:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787829711.718867:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787829711.718868:0:40798:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4512 00000100:00080000:2.0:1787829711.718870:0:40798:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787829711.718872:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829711.718872:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787829711.718874:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787829711.718875:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787829711.718877:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000020:00000001:2.0:1787829711.718879:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:2.0:1787829711.718880:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829711.718881:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:2.0:1787829711.718884:0:40798:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787829711.718887:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829711.718888:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787829711.718893:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787829711.718894:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829711.718895:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:3.0:1787829711.718898:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000008:00100000:2.0:1787829711.718899:0:40798:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1787829711.718903:0:40798:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829711.718905:0:40798:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829711.718905:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 02000000:00000001:2.0:1787829711.718911:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:3.0:1787829711.718911:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 02000000:00000001:2.0:1787829711.718915:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829711.718915:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787829711.718917:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:3.0:1787829711.718918:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829711.718919:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787829711.718923:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829711.718925:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:2.0:1787829711.718928:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829711.718929:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00100000:2.0:1787829711.718934:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2cb40 x1874670646857216/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:3.0:1787829711.718943:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829711.718944:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787829711.718948:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829711.718952:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787829711.718956:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1787829711.718956:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646857088:192.168.203.143@tcp:250:kworker.0 00000100:00080000:2.0:1787829711.718962:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1787829711.718966:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787829711.718968:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787829711.718968:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:2.0:1787829711.718973:0:40798:0:(import.c:485:import_select_connection()) Process entered 02000000:00000001:3.0:1787829711.718975:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000100:00080000:2.0:1787829711.718976:0:40798:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4512 02000000:00000001:3.0:1787829711.718980:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787829711.718981:0:40798:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787829711.718985:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829711.718988:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 02000000:00000001:3.0:1787829711.718988:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000020:00000001:2.0:1787829711.718992:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787829711.718993:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787829711.718994:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000020:00000001:2.0:1787829711.718995:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 02000000:00000001:3.0:1787829711.718996:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829711.718997:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829711.719000:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:2.0:1787829711.719003:0:40798:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787829711.719005:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829711.719006:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787829711.719008:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787829711.719009:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829711.719011:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1787829711.719012:0:40798:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1787829711.719014:0:40798:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829711.719014:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787829711.719015:0:40798:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829711.719019:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417869248 : -126234291682368 : ffff8d30c9e2c3c0) 02000000:00000001:2.0:1787829711.719021:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787829711.719025:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829711.719026:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787829711.719028:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:3.0:1787829711.719028:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 02000000:00000001:2.0:1787829711.719032:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829711.719036:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787829711.719039:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2fc00 x1874670646857344/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787829711.719044:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787829711.719046:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829711.719048:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787829711.719050:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829711.719051:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1787829711.719052:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787829711.719053:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787829711.719055:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787829711.719057:0:40798:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4512 00000100:00080000:2.0:1787829711.719060:0:40798:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787829711.719062:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829711.719063:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787829711.719064:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787829711.719065:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787829711.719066:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000020:00000001:2.0:1787829711.719068:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000800:00000100:3.0:1787829711.719072:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:2.0:1787829711.719079:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829711.719081:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:3.0:1787829711.719084:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00080000:2.0:1787829711.719088:0:40798:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:3.0:1787829711.719090:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829711.719093:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829711.719094:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829711.719097:0:2390:0:(events.c:68:request_out_callback()) Process leaving 02000000:00000001:2.0:1787829711.719098:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787829711.719102:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829711.719103:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829711.719104:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829711.719109:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1787829711.719111:0:40798:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00000001:3.0:1787829711.719111:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4299185920 00000008:00000001:2.0:1787829711.719116:0:40798:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829711.719118:0:40798:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829711.719122:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787829711.719123:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829711.719124:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 00000100:00000001:2.0:1787829711.719125:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787829711.719128:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787829711.719130:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829711.719133:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4299185920 00000100:00000001:2.0:1787829711.719134:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787829711.719142:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2c780 x1874670646857472/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787829711.719142:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 00000100:00000001:3.0:1787829711.719147:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829711.719164:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829711.719167:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787829711.719168:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00000001:2.0:1787829711.719170:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829711.719174:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829711.719177:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829711/real 1787829711] req@ffff8d30c9e2c3c0 x1874670646857088/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829736 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1787829711.719187:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787829711.719188:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829711.719192:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829711.719195:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829711.719197:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829711.719199:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829711.719201:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829711.719206:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2c3c0 x1874670646857088/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829736 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829711.719221:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829711.719224:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829711.719228:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829711.719229:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829711.719231:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829711.719233:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829711.719236:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829711.719239:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829711.719244:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829711.719247:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829711.719251:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2c3c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646857088:192.168.203.143@tcp:250:kworker.0 00000100:00000001:3.0:1787829711.719255:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829711.719257:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829711.719258:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829711.719260:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829711.719262:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829711.719264:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829711.719265:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829711.719268:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829711.719269:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829711.719272:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829711.719273:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829711.719274:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829711.719281:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787829711.719282:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829711.719284:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1787829711.719285:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787829711.719286:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787829711.719287:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787829711.719288:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829711.719291:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787829711.719293:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829711.719295:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829711.719299:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646857216:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829711.719302:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1787829711.719305:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1787829711.719306:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829711.719308:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1787829711.719311:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829711.719317:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787829711.719319:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417871168 : -126234291680448 : ffff8d30c9e2cb40) 00000100:00000001:3.0:1787829711.719324:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1787829711.719338:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:3.0:1787829711.719344:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787829711.719345:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829711.719346:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829711.719348:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1787829711.719350:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829711.719353:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4299184662 00000100:00000001:3.0:1787829711.719355:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4299185920 00000100:00000001:3.0:1787829711.719357:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4299184662 00000100:00000001:3.0:1787829711.719358:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4299185920 00000100:00000001:3.0:1787829711.719360:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829711.719361:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829711.719362:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1787829711.719366:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829711/real 1787829711] req@ffff8d30c9e2cb40 x1874670646857216/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829736 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1787829711.719371:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787829711.719372:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829711.719374:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829711.719375:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829711.719376:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829711.719377:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829711.719378:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829711.719381:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2cb40 x1874670646857216/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829736 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829711.719390:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829711.719392:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829711.719394:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829711.719395:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829711.719396:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829711.719397:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829711.719398:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829711.719399:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829711.719401:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829711.719403:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829711.719405:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2cb40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646857216:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829711.719407:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787829711.719408:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787829711.719409:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829711.719410:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787829711.719412:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829711.719413:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829711.719415:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646857344:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829711.719417:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1787829711.719419:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1787829711.719419:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829711.719421:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1787829711.719423:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829711.719425:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787829711.719426:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417883648 : -126234291667968 : ffff8d30c9e2fc00) 00000100:00000001:3.0:1787829711.719429:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1787829711.719437:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:3.0:1787829711.719439:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787829711.719441:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829711.719442:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829711.719443:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1787829711.719444:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829711.719447:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4299184662 00000100:00000001:3.0:1787829711.719448:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4299185920 00000100:00000001:3.0:1787829711.719450:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4299184662 00000100:00000001:3.0:1787829711.719451:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4299185920 00000100:00000001:3.0:1787829711.719453:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829711.719454:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829711.719455:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1787829711.719457:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829711/real 1787829711] req@ffff8d30c9e2fc00 x1874670646857344/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829736 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1787829711.719462:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787829711.719463:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829711.719464:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829711.719465:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829711.719466:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829711.719467:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829711.719467:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829711.719470:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2fc00 x1874670646857344/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829736 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829711.719474:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829711.719475:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829711.719477:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829711.719478:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829711.719479:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829711.719480:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829711.719481:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829711.719482:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829711.719484:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829711.719485:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829711.719488:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646857344:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829711.719490:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787829711.719491:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787829711.719492:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829711.719493:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787829711.719494:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829711.719496:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829711.719497:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646857472:192.168.203.143@tcp:8:kworker.0 00000100:00000001:3.0:1787829711.719499:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1787829711.719500:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1787829711.719501:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829711.719503:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1787829711.719505:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829711.719511:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787829711.719513:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417870208 : -126234291681408 : ffff8d30c9e2c780) 00000100:00000001:3.0:1787829711.719516:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1787829711.719526:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:3.0:1787829711.719531:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787829711.719533:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829711.719534:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829711.719536:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1787829711.719538:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829711.719542:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4299184675 00000100:00000001:3.0:1787829711.719545:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4299185920 00000100:00000001:3.0:1787829711.719547:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4299184675 00000100:00000001:3.0:1787829711.719549:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4299185920 00000100:00000001:3.0:1787829711.719551:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829711.719554:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829711.719555:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1787829711.719559:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829711/real 1787829711] req@ffff8d30c9e2c780 x1874670646857472/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829736 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1787829711.719570:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787829711.719571:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829711.719580:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829711.719581:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829711.719582:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829711.719583:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829711.719584:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829711.719587:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2c780 x1874670646857472/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829736 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829711.719592:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829711.719593:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829711.719595:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829711.719596:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829711.719596:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829711.719598:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829711.719626:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829711.719628:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829711.719630:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829711.719631:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829711.719633:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646857472:192.168.203.143@tcp:8:kworker.0 00000100:00000001:3.0:1787829711.719636:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829711.719637:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829711.719638:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829711.719639:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829711.719641:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829711.719642:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829711.719643:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829711.719644:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829711.719645:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829711.719647:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829711.719648:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829711.719650:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829711.719650:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829711.719651:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829711.719652:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829711.719653:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829711.719654:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829711.719655:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829711.719656:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829711.719658:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829711.719659:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829711.719659:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829711.719660:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829711.719661:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829711.719662:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829711.719662:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829711.719663:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829711.719664:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829711.719664:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829711.719665:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829711.719666:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829711.719667:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829711.719669:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787829711.719670:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829711.719672:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1787829711.719673:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829712.614722:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829712.614728:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829712.614788:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829712.614792:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829712.614795:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829713.638721:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829713.638727:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829713.638774:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829713.638778:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829713.638782:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829714.662689:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829714.662694:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829714.662756:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829714.662760:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829714.662764:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829715.686688:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829715.686696:0:815:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1787829715.686698:0:815:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1787829715.686702:0:815:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829715.686707:0:815:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1787829715.686709:0:815:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829715.686711:0:815:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829715.686714:0:815:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829715.686757:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829715.686760:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829715.686763:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829716.710705:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829716.710710:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829716.710772:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829716.710775:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829716.710778:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787829716.838733:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829716.838743:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1787829716.838746:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787829716.838749:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787829716.838754:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787829716.838761:0:40798:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 4517 00000100:00080000:0.0:1787829716.838772:0:40798:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787829716.838775:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829716.838777:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787829716.838781:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787829716.838783:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787829716.838786:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:0.0:1787829716.838787:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:0.0:1787829716.838789:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829716.838790:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:0.0:1787829716.838795:0:40798:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787829716.838797:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829716.838799:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787829716.838803:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787829716.838806:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829716.838808:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1787829716.838810:0:40798:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829716.838823:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787829716.838824:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829716.838827:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787829716.838830:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787829716.838835:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829716.838847:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787829716.838856:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d25a40 x1874670646857600/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787829716.838866:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787829716.838870:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829716.838886:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787829716.838888:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829716.838895:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1787829716.838896:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787829716.838897:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787829716.838900:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787829716.838902:0:40798:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4517 00000100:00080000:0.0:1787829716.838904:0:40798:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787829716.838906:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829716.838907:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787829716.838909:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787829716.838910:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787829716.838912:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000020:00000001:0.0:1787829716.838919:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:0.0:1787829716.838921:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829716.838922:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:0.0:1787829716.838925:0:40798:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:3.0:1787829716.838926:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787829716.838927:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829716.838931:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:3.0:1787829716.838933:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787829716.838936:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 02000000:00000001:0.0:1787829716.838937:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829716.838939:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787829716.838943:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829716.838944:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829716.838948:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 02000000:00000001:3.0:1787829716.838949:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787829716.838953:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000008:00100000:0.0:1787829716.838954:0:40798:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 02000000:00000001:3.0:1787829716.838956:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:0.0:1787829716.838962:0:40798:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829716.838965:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d25a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646857600:192.168.203.143@tcp:250:kworker.0 00000100:00000001:0.0:1787829716.838967:0:40798:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829716.838975:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:3.0:1787829716.838976:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787829716.838979:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829716.838983:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787829716.838983:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1787829716.838987:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829716.838988:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787829716.838993:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829716.838993:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1787829716.839000:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829716.839002:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787829716.839010:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d24f00 x1874670646857728/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787829716.839017:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787829716.839018:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829716.839021:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829716.839022:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00080000:0.0:1787829716.839024:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1787829716.839025:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601343040 : -126234108208576 : ffff8d30d4d25a40) 00000100:00080000:0.0:1787829716.839031:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1787829716.839033:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:0.0:1787829716.839034:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787829716.839037:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787829716.839039:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787829716.839041:0:40798:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4517 00000100:00080000:0.0:1787829716.839044:0:40798:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787829716.839045:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829716.839046:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787829716.839048:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787829716.839050:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787829716.839052:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000020:00000001:0.0:1787829716.839054:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:0.0:1787829716.839055:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829716.839056:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:0.0:1787829716.839058:0:40798:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787829716.839060:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829716.839061:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787829716.839063:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787829716.839065:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829716.839066:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1787829716.839067:0:40798:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1787829716.839069:0:40798:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829716.839070:0:40798:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829716.839074:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787829716.839076:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:3.0:1787829716.839078:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:0.0:1787829716.839079:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787829716.839081:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787829716.839084:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829716.839087:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787829716.839092:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d24000 x1874670646857856/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787829716.839095:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1787829716.839100:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787829716.839102:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829716.839102:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787829716.839107:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829716.839107:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787829716.839110:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1787829716.839111:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00080000:0.0:1787829716.839132:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1787829716.839133:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787829716.839135:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787829716.839137:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:3.0:1787829716.839137:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787829716.839142:0:40798:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4517 00000100:00000001:3.0:1787829716.839145:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4299191040 00000100:00080000:0.0:1787829716.839147:0:40798:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787829716.839150:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829716.839151:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787829716.839153:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787829716.839154:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787829716.839155:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000020:00000001:0.0:1787829716.839157:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:3.0:1787829716.839157:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1787829716.839159:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829716.839162:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:0.0:1787829716.839165:0:40798:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:3.0:1787829716.839168:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4299191040 00000100:00000001:0.0:1787829716.839169:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829716.839171:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787829716.839176:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829716.839176:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 02000000:00000001:0.0:1787829716.839178:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829716.839181:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:3.0:1787829716.839182:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000008:00100000:0.0:1787829716.839183:0:40798:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1787829716.839189:0:40798:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829716.839189:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829716.839194:0:40798:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829716.839196:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 02000000:00000001:0.0:1787829716.839199:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787829716.839202:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829716.839204:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829716/real 1787829716] req@ffff8d30d4d25a40 x1874670646857600/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829741 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787829716.839205:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787829716.839208:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787829716.839210:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829716.839217:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:3.0:1787829716.839220:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00100000:0.0:1787829716.839224:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d270c0 x1874670646857984/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000001:3.0:1787829716.839225:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829716.839230:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787829716.839233:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787829716.839235:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787829716.839236:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829716.839238:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829716.839239:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829716.839241:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829716.839244:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829716.839250:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d25a40 x1874670646857600/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829741 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829716.839256:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829716.839260:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829716.839262:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829716.839264:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829716.839265:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829716.839275:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829716.839277:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829716.839279:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829716.839283:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829716.839284:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829716.839287:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d25a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646857600:192.168.203.143@tcp:250:kworker.0 00000100:00000001:3.0:1787829716.839291:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829716.839293:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829716.839294:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829716.839297:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829716.839300:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829716.839302:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829716.839304:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829716.839309:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829716.839310:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829716.839313:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829716.839314:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829716.839316:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829716.839323:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787829716.839325:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829716.839327:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1787829716.839328:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787829716.839330:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787829716.839331:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787829716.839333:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829716.839335:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787829716.839337:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829716.839340:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829716.839343:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d24f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646857728:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829716.839347:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1787829716.839349:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1787829716.839352:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829716.839354:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1787829716.839357:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829716.839364:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787829716.839365:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601340160 : -126234108211456 : ffff8d30d4d24f00) 00000100:00000001:3.0:1787829716.839372:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1787829716.839386:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:3.0:1787829716.839390:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787829716.839392:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829716.839393:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829716.839394:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1787829716.839396:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829716.839401:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4299189782 00000100:00000001:3.0:1787829716.839403:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4299191040 00000100:00000001:3.0:1787829716.839405:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4299189782 00000100:00000001:3.0:1787829716.839407:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4299191040 00000100:00000001:3.0:1787829716.839409:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829716.839410:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829716.839411:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1787829716.839414:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829716/real 1787829716] req@ffff8d30d4d24f00 x1874670646857728/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829741 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1787829716.839420:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787829716.839421:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829716.839422:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829716.839424:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829716.839425:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829716.839426:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829716.839427:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829716.839429:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d24f00 x1874670646857728/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829741 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829716.839434:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829716.839435:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829716.839438:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829716.839439:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829716.839440:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829716.839441:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829716.839443:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829716.839444:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829716.839446:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829716.839448:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829716.839449:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d24f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646857728:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829716.839452:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787829716.839453:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787829716.839454:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829716.839455:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787829716.839456:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829716.839457:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829716.839459:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d24000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646857856:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829716.839462:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1787829716.839463:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1787829716.839464:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829716.839465:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1787829716.839467:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829716.839470:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787829716.839471:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601336320 : -126234108215296 : ffff8d30d4d24000) 00000100:00000001:3.0:1787829716.839474:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1787829716.839481:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:3.0:1787829716.839484:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787829716.839485:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829716.839487:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829716.839488:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1787829716.839489:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829716.839492:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4299189782 00000100:00000001:3.0:1787829716.839493:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4299191040 00000100:00000001:3.0:1787829716.839495:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4299189782 00000100:00000001:3.0:1787829716.839497:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4299191040 00000100:00000001:3.0:1787829716.839499:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829716.839501:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829716.839502:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1787829716.839504:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829716/real 1787829716] req@ffff8d30d4d24000 x1874670646857856/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829741 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1787829716.839508:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787829716.839509:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829716.839510:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829716.839511:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829716.839512:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829716.839513:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829716.839514:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829716.839516:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d24000 x1874670646857856/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829741 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829716.839521:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829716.839522:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829716.839524:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829716.839525:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829716.839526:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829716.839527:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829716.839528:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829716.839529:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829716.839531:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829716.839532:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829716.839534:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d24000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646857856:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829716.839536:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787829716.839537:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787829716.839539:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829716.839540:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787829716.839541:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829716.839543:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829716.839544:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d270c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646857984:192.168.203.143@tcp:8:kworker.0 00000100:00000001:3.0:1787829716.839546:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1787829716.839547:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1787829716.839548:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829716.839549:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1787829716.839551:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829716.839554:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787829716.839555:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601348800 : -126234108202816 : ffff8d30d4d270c0) 00000100:00000001:3.0:1787829716.839557:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1787829716.839563:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:3.0:1787829716.839565:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787829716.839567:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829716.839568:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829716.839577:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1787829716.839579:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829716.839581:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4299189795 00000100:00000001:3.0:1787829716.839583:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4299191040 00000100:00000001:3.0:1787829716.839584:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4299189795 00000100:00000001:3.0:1787829716.839586:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4299191040 00000100:00000001:3.0:1787829716.839587:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829716.839589:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829716.839590:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1787829716.839592:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829716/real 1787829716] req@ffff8d30d4d270c0 x1874670646857984/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829741 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1787829716.839597:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787829716.839597:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829716.839639:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829716.839641:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829716.839642:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829716.839644:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829716.839645:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829716.839648:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d270c0 x1874670646857984/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829741 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829716.839653:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829716.839655:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829716.839657:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829716.839658:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829716.839658:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829716.839659:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829716.839661:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829716.839662:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829716.839664:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829716.839665:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829716.839667:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d270c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646857984:192.168.203.143@tcp:8:kworker.0 00000100:00000001:3.0:1787829716.839670:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829716.839671:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829716.839672:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829716.839673:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829716.839675:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829716.839676:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829716.839676:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829716.839678:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829716.839679:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829716.839682:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829716.839682:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829716.839684:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829716.839684:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829716.839685:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829716.839687:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829716.839687:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829716.839688:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829716.839690:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829716.839691:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829716.839692:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829716.839694:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829716.839696:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829716.839697:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829716.839698:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829716.839700:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829716.839701:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829716.839702:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829716.839704:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829716.839705:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829716.839708:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829716.839710:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829716.839712:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829716.839715:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787829716.839717:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829716.839719:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1787829716.839721:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829717.734676:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829717.734684:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829717.734762:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829717.734767:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829717.734772:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829718.758737:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829718.758746:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829718.758802:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829718.758807:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829718.758811:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829719.782763:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829719.782773:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829719.782861:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829719.782866:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829719.782870:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829720.806733:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829720.806741:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829720.806803:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829720.806809:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829720.806814:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829721.830757:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829721.830767:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829721.830879:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829721.830886:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829721.830891:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787829721.958730:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829721.958740:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1787829721.958744:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787829721.958748:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787829721.958753:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787829721.958763:0:40798:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 4522 00000100:00080000:2.0:1787829721.958788:0:40798:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787829721.958793:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829721.958796:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787829721.958802:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787829721.958806:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787829721.958810:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:2.0:1787829721.958813:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:2.0:1787829721.958815:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829721.958817:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:2.0:1787829721.958822:0:40798:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787829721.958824:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829721.958827:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787829721.958831:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787829721.958834:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829721.958837:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:2.0:1787829721.958840:0:40798:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829721.958853:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787829721.958855:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829721.958859:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787829721.958862:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787829721.958866:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829721.958880:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787829721.958890:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2f0c0 x1874670646858112/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787829721.958900:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787829721.958905:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829721.958922:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787829721.958927:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829721.958930:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1787829721.958933:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787829721.958934:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787829721.958938:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787829721.958943:0:40798:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4522 00000100:00080000:2.0:1787829721.958949:0:40798:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787829721.958951:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829721.958952:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787829721.958954:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787829721.958956:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:3.0:1787829721.958958:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:2.0:1787829721.958959:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000020:00000001:2.0:1787829721.958963:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:2.0:1787829721.958965:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829721.958966:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1787829721.958970:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:3.0:1787829721.958972:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00080000:2.0:1787829721.958979:0:40798:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:3.0:1787829721.958980:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787829721.958989:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829721.958990:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829721.959004:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787829721.959007:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787829721.959011:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787829721.959012:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787829721.959018:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829721.959019:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1787829721.959024:0:40798:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1787829721.959027:0:40798:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829721.959028:0:40798:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829721.959031:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787829721.959032:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829721.959033:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787829721.959034:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787829721.959035:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829721.959041:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787829721.959045:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2ed00 x1874670646858240/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787829721.959051:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787829721.959052:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829721.959055:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787829721.959057:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829721.959060:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1787829721.959062:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787829721.959063:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787829721.959064:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787829721.959066:0:40798:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4522 00000100:00080000:2.0:1787829721.959068:0:40798:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787829721.959070:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829721.959071:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787829721.959073:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787829721.959075:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 02000000:00000001:3.0:1787829721.959089:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1787829721.959090:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000020:00000001:2.0:1787829721.959098:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:2.0:1787829721.959101:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829721.959106:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00100000:3.0:1787829721.959107:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646858112:192.168.203.143@tcp:250:kworker.0 00000100:00080000:2.0:1787829721.959118:0:40798:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:3.0:1787829721.959118:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:2.0:1787829721.959123:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829721.959124:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1787829721.959126:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787829721.959128:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829721.959130:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829721.959132:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1787829721.959134:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829721.959137:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 02000000:00000001:3.0:1787829721.959138:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000008:00100000:2.0:1787829721.959142:0:40798:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1787829721.959146:0:40798:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829721.959147:0:40798:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829721.959150:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787829721.959151:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829721.959152:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787829721.959153:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787829721.959155:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829721.959159:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787829721.959163:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2e940 x1874670646858368/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787829721.959169:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787829721.959170:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829721.959172:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787829721.959174:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829721.959176:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1787829721.959178:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787829721.959179:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787829721.959181:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787829721.959183:0:40798:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4522 00000100:00080000:2.0:1787829721.959186:0:40798:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787829721.959187:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829721.959189:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787829721.959190:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787829721.959191:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787829721.959193:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000020:00000001:2.0:1787829721.959194:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:2.0:1787829721.959195:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829721.959197:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:2.0:1787829721.959199:0:40798:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787829721.959200:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829721.959201:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787829721.959203:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787829721.959204:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829721.959205:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1787829721.959207:0:40798:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000008:00000001:2.0:1787829721.959209:0:40798:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829721.959210:0:40798:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829721.959213:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787829721.959213:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829721.959214:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787829721.959215:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787829721.959216:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829721.959222:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787829721.959225:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2c000 x1874670646858496/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787829721.959230:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787829721.959232:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829721.959236:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829721.959252:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787829721.959255:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417880768 : -126234291670848 : ffff8d30c9e2f0c0) 00000100:00000001:3.0:1787829721.959262:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1787829721.959388:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:3.0:1787829721.959397:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787829721.959401:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829721.959403:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829721.959404:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1787829721.959409:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829721.959412:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4299196160 00000100:00000001:3.0:1787829721.959419:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 00000100:00000001:3.0:1787829721.959423:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4299196160 00000100:00000001:3.0:1787829721.959426:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 00000100:00000001:3.0:1787829721.959429:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829721.959431:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829721.959433:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1787829721.959439:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829721/real 1787829721] req@ffff8d30c9e2f0c0 x1874670646858112/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829746 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1787829721.959447:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787829721.959448:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829721.959452:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829721.959454:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829721.959456:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829721.959459:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829721.959461:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829721.959464:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2f0c0 x1874670646858112/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829746 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829721.959470:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829721.959473:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829721.959475:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829721.959476:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829721.959477:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829721.959479:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829721.959481:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829721.959483:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829721.959486:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829721.959488:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829721.959491:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646858112:192.168.203.143@tcp:250:kworker.0 00000100:00000001:3.0:1787829721.959495:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829721.959497:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829721.959498:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829721.959500:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829721.959502:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829721.959504:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829721.959506:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829721.959509:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829721.959510:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829721.959513:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829721.959514:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829721.959516:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829721.959523:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787829721.959525:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829721.959528:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1787829721.959530:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787829721.959531:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787829721.959532:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787829721.959533:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829721.959535:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787829721.959537:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829721.959539:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829721.959541:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646858240:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829721.959543:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1787829721.959545:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1787829721.959546:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829721.959548:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1787829721.959549:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829721.959554:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787829721.959555:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417879808 : -126234291671808 : ffff8d30c9e2ed00) 00000100:00000001:3.0:1787829721.959559:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1787829721.959569:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:3.0:1787829721.959572:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787829721.959574:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829721.959575:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829721.959576:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1787829721.959578:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829721.959581:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4299194902 00000100:00000001:3.0:1787829721.959583:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4299196160 00000100:00000001:3.0:1787829721.959585:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4299194902 00000100:00000001:3.0:1787829721.959586:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4299196160 00000100:00000001:3.0:1787829721.959588:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829721.959589:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829721.959590:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1787829721.959594:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829721/real 1787829721] req@ffff8d30c9e2ed00 x1874670646858240/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829746 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1787829721.959645:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787829721.959647:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829721.959651:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829721.959654:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829721.959656:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829721.959659:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829721.959661:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829721.959666:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2ed00 x1874670646858240/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829746 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829721.959675:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829721.959677:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829721.959679:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829721.959680:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829721.959681:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829721.959683:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829721.959684:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829721.959686:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829721.959689:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829721.959690:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829721.959693:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646858240:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829721.959696:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787829721.959697:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787829721.959698:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829721.959700:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787829721.959701:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829721.959703:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829721.959705:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646858368:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829721.959707:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1787829721.959709:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1787829721.959710:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829721.959711:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1787829721.959713:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829721.959717:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787829721.959718:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417878848 : -126234291672768 : ffff8d30c9e2e940) 00000100:00000001:3.0:1787829721.959721:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1787829721.959731:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:3.0:1787829721.959735:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787829721.959736:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829721.959737:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829721.959739:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1787829721.959741:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829721.959744:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4299194903 00000100:00000001:3.0:1787829721.959747:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4299196160 00000100:00000001:3.0:1787829721.959749:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4299194903 00000100:00000001:3.0:1787829721.959751:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4299196160 00000100:00000001:3.0:1787829721.959753:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829721.959755:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829721.959756:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1787829721.959759:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829721/real 1787829721] req@ffff8d30c9e2e940 x1874670646858368/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829746 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1787829721.959770:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787829721.959772:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829721.959774:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829721.959776:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829721.959777:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829721.959779:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829721.959781:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829721.959784:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2e940 x1874670646858368/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829746 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829721.959793:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829721.959795:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829721.959798:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829721.959800:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829721.959801:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829721.959803:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829721.959805:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829721.959808:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829721.959811:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829721.959814:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829721.959818:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646858368:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829721.959823:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787829721.959824:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787829721.959825:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829721.959827:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787829721.959829:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829721.959830:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829721.959832:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646858496:192.168.203.143@tcp:8:kworker.0 00000100:00000001:3.0:1787829721.959835:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1787829721.959837:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1787829721.959838:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829721.959839:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1787829721.959842:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829721.959846:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787829721.959847:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417868288 : -126234291683328 : ffff8d30c9e2c000) 00000100:00000001:3.0:1787829721.959850:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1787829721.959859:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:3.0:1787829721.959862:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787829721.959863:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829721.959865:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829721.959866:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1787829721.959867:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829721.959870:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4299194916 00000100:00000001:3.0:1787829721.959871:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4299196160 00000100:00000001:3.0:1787829721.959882:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4299194916 00000100:00000001:3.0:1787829721.959883:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4299196160 00000100:00000001:3.0:1787829721.959885:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829721.959886:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829721.959887:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1787829721.959890:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829721/real 1787829721] req@ffff8d30c9e2c000 x1874670646858496/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829746 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1787829721.959898:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787829721.959899:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829721.959902:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829721.959904:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829721.959905:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829721.959907:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829721.959909:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829721.959913:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2c000 x1874670646858496/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829746 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829721.959920:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829721.959923:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829721.959926:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829721.959928:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829721.959929:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829721.959931:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829721.959932:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829721.959935:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829721.959939:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829721.959941:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829721.959943:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646858496:192.168.203.143@tcp:8:kworker.0 00000100:00000001:3.0:1787829721.959946:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829721.959947:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829721.959948:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829721.959949:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829721.959951:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829721.959952:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829721.959953:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829721.959954:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829721.959955:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829721.959958:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829721.959959:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829721.959960:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829721.959961:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829721.959962:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829721.959963:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829721.959964:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829721.959965:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829721.959966:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829721.959966:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829721.959967:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829721.959968:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829721.959969:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829721.959970:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829721.959970:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829721.959971:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829721.959972:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829721.959972:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829721.959973:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829721.959974:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829721.959976:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829721.959977:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829721.959978:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829721.959981:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787829721.959982:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829721.959983:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1787829721.959984:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829722.854677:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829722.854682:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829722.854740:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829722.854743:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829722.854746:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829723.878712:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829723.878718:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829723.878812:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829723.878819:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829723.878825:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829724.902664:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829724.902670:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829724.902710:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829724.902714:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829724.902716:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829725.926718:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829725.926725:0:815:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1787829725.926726:0:815:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1787829725.926729:0:815:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829725.926733:0:815:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1787829725.926734:0:815:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829725.926736:0:815:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829725.926738:0:815:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829725.926820:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829725.926824:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829725.926826:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829726.950713:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829726.950720:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829726.950786:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829726.950791:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829726.950795:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787829727.078724:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829727.078731:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1787829727.078733:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787829727.078736:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787829727.078740:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787829727.078745:0:40798:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 4527 00000100:00080000:0.0:1787829727.078754:0:40798:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787829727.078756:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829727.078757:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787829727.078761:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787829727.078763:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787829727.078765:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:0.0:1787829727.078767:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:0.0:1787829727.078768:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829727.078769:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:0.0:1787829727.078773:0:40798:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787829727.078775:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829727.078776:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787829727.078779:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787829727.078781:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829727.078783:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1787829727.078784:0:40798:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829727.078794:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787829727.078795:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829727.078797:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787829727.078800:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787829727.078803:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829727.078814:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787829727.078820:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d270c0 x1874670646858624/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787829727.078828:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787829727.078831:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829727.078843:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787829727.078845:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829727.078847:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1787829727.078848:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787829727.078850:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787829727.078852:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787829727.078853:0:40798:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4527 00000100:00080000:0.0:1787829727.078855:0:40798:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787829727.078857:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829727.078858:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787829727.078861:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787829727.078862:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787829727.078864:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000020:00000001:0.0:1787829727.078867:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:0.0:1787829727.078870:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829727.078872:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:0.0:1787829727.078875:0:40798:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787829727.078878:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829727.078880:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:3.0:1787829727.078882:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:0.0:1787829727.078886:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829727.078888:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 02000000:00000001:0.0:1787829727.078892:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829727.078893:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1787829727.078896:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:3.0:1787829727.078898:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787829727.078901:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000008:00100000:0.0:1787829727.078903:0:40798:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 02000000:00000001:3.0:1787829727.078906:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000008:00000001:0.0:1787829727.078908:0:40798:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829727.078911:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787829727.078913:0:40798:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829727.078916:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829727.078920:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787829727.078923:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829727.078924:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d270c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646858624:192.168.203.143@tcp:250:kworker.0 00000100:00000001:0.0:1787829727.078925:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787829727.078928:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787829727.078930:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829727.078934:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:0.0:1787829727.078936:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829727.078940:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000100:00100000:0.0:1787829727.078944:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d24000 x1874670646858752/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:3.0:1787829727.078945:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829727.078950:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:0.0:1787829727.078955:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787829727.078957:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829727.078957:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829727.078960:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787829727.078964:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829727.078966:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1787829727.078968:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787829727.078969:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787829727.078971:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787829727.078972:0:40798:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4527 00000100:00000001:3.0:1787829727.078974:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00080000:0.0:1787829727.078976:0:40798:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787829727.078979:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829727.078979:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601348800 : -126234108202816 : ffff8d30d4d270c0) 00000100:00000001:0.0:1787829727.078982:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787829727.078986:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:3.0:1787829727.078986:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000020:00000001:0.0:1787829727.078988:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787829727.078991:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000020:00000001:0.0:1787829727.078993:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:0.0:1787829727.078994:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829727.078995:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:0.0:1787829727.078997:0:40798:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787829727.078999:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829727.079000:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787829727.079002:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787829727.079004:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829727.079005:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1787829727.079006:0:40798:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1787829727.079007:0:40798:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829727.079009:0:40798:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829727.079012:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787829727.079013:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829727.079014:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787829727.079015:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787829727.079016:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829727.079020:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787829727.079023:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d24f00 x1874670646858880/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1787829727.079028:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:0.0:1787829727.079037:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787829727.079039:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829727.079053:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829727.079055:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00080000:0.0:1787829727.079057:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1787829727.079061:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00080000:0.0:1787829727.079062:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1787829727.079065:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:3.0:1787829727.079066:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787829727.079067:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787829727.079071:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787829727.079072:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787829727.079077:0:40798:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4527 00000100:00000001:3.0:1787829727.079080:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787829727.079081:0:40798:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787829727.079085:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829727.079087:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:3.0:1787829727.079087:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4299201280 00000020:00000001:0.0:1787829727.079090:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787829727.079093:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787829727.079094:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000020:00000001:0.0:1787829727.079096:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:0.0:1787829727.079097:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829727.079098:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:3.0:1787829727.079100:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 00000100:00080000:0.0:1787829727.079101:0:40798:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787829727.079105:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829727.079106:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787829727.079109:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829727.079110:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4299201280 02000000:00000001:0.0:1787829727.079111:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829727.079114:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1787829727.079117:0:40798:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00000001:3.0:1787829727.079118:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 00000008:00000001:0.0:1787829727.079119:0:40798:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829727.079124:0:40798:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829727.079126:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787829727.079128:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787829727.079130:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829727.079132:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829727.079133:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787829727.079135:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787829727.079139:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829727.079139:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00000001:0.0:1787829727.079146:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829727.079149:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829727/real 1787829727] req@ffff8d30d4d270c0 x1874670646858624/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829752 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829727.079152:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d25a40 x1874670646859008/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000400:00000001:3.0:1787829727.079171:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787829727.079174:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787829727.079179:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787829727.079180:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787829727.079183:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829727.079185:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787829727.079187:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829727.079189:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829727.079193:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829727.079195:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829727.079202:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d270c0 x1874670646858624/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829752 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829727.079212:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829727.079215:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829727.079218:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829727.079220:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829727.079221:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829727.079223:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829727.079225:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829727.079229:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829727.079233:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829727.079236:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829727.079240:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d270c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646858624:192.168.203.143@tcp:250:kworker.0 00000100:00000001:3.0:1787829727.079245:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829727.079247:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829727.079249:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829727.079252:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829727.079255:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829727.079257:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829727.079259:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829727.079264:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829727.079265:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829727.079269:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829727.079270:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829727.079273:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829727.079279:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787829727.079281:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829727.079283:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1787829727.079285:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787829727.079286:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787829727.079288:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787829727.079290:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829727.079293:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787829727.079295:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829727.079297:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829727.079300:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d24000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646858752:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829727.079304:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1787829727.079306:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1787829727.079308:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829727.079309:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1787829727.079312:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829727.079318:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787829727.079319:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601336320 : -126234108215296 : ffff8d30d4d24000) 00000100:00000001:3.0:1787829727.079324:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1787829727.079336:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:3.0:1787829727.079340:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787829727.079342:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829727.079344:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829727.079346:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1787829727.079349:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829727.079353:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4299200022 00000100:00000001:3.0:1787829727.079356:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4299201280 00000100:00000001:3.0:1787829727.079359:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4299200022 00000100:00000001:3.0:1787829727.079361:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4299201280 00000100:00000001:3.0:1787829727.079364:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829727.079366:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829727.079368:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1787829727.079372:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829727/real 1787829727] req@ffff8d30d4d24000 x1874670646858752/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829752 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1787829727.079381:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787829727.079383:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829727.079385:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829727.079387:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829727.079388:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829727.079390:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829727.079391:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829727.079395:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d24000 x1874670646858752/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829752 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829727.079403:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829727.079406:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829727.079408:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829727.079410:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829727.079411:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829727.079413:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829727.079415:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829727.079418:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829727.079421:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829727.079423:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829727.079427:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d24000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646858752:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829727.079430:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787829727.079432:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787829727.079433:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829727.079435:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787829727.079437:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829727.079447:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829727.079451:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d24f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646858880:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829727.079454:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1787829727.079456:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1787829727.079458:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829727.079460:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1787829727.079462:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829727.079467:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787829727.079468:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601340160 : -126234108211456 : ffff8d30d4d24f00) 00000100:00000001:3.0:1787829727.079472:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1787829727.079482:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:3.0:1787829727.079486:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787829727.079488:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829727.079489:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829727.079491:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1787829727.079493:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829727.079497:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4299200022 00000100:00000001:3.0:1787829727.079500:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4299201280 00000100:00000001:3.0:1787829727.079502:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4299200022 00000100:00000001:3.0:1787829727.079505:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4299201280 00000100:00000001:3.0:1787829727.079507:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829727.079509:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829727.079511:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1787829727.079515:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829727/real 1787829727] req@ffff8d30d4d24f00 x1874670646858880/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829752 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1787829727.079522:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787829727.079524:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829727.079526:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829727.079528:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829727.079530:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829727.079531:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829727.079533:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829727.079537:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d24f00 x1874670646858880/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829752 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829727.079551:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829727.079554:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829727.079556:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829727.079557:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829727.079559:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829727.079560:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829727.079562:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829727.079565:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829727.079568:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829727.079570:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829727.079574:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d24f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646858880:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829727.079577:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787829727.079579:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787829727.079580:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829727.079582:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787829727.079584:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829727.079586:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829727.079589:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d25a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646859008:192.168.203.143@tcp:8:kworker.0 00000100:00000001:3.0:1787829727.079593:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1787829727.079594:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1787829727.079596:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829727.079598:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1787829727.079630:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829727.079635:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787829727.079637:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601343040 : -126234108208576 : ffff8d30d4d25a40) 00000100:00000001:3.0:1787829727.079640:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1787829727.079651:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:3.0:1787829727.079655:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787829727.079658:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829727.079659:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829727.079661:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1787829727.079663:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829727.079667:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4299200036 00000100:00000001:3.0:1787829727.079670:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4299201280 00000100:00000001:3.0:1787829727.079672:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4299200036 00000100:00000001:3.0:1787829727.079674:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4299201280 00000100:00000001:3.0:1787829727.079677:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829727.079679:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829727.079681:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1787829727.079685:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829727/real 1787829727] req@ffff8d30d4d25a40 x1874670646859008/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829752 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1787829727.079694:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787829727.079696:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829727.079698:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829727.079700:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829727.079701:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829727.079703:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829727.079704:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829727.079708:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d25a40 x1874670646859008/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829752 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829727.079715:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829727.079717:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829727.079719:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829727.079721:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829727.079722:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829727.079724:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829727.079725:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829727.079728:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829727.079731:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829727.079733:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829727.079736:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d25a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646859008:192.168.203.143@tcp:8:kworker.0 00000100:00000001:3.0:1787829727.079740:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829727.079742:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829727.079743:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829727.079745:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829727.079747:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829727.079749:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829727.079750:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829727.079752:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829727.079754:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829727.079756:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829727.079757:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829727.079759:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829727.079760:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829727.079761:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829727.079763:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829727.079764:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829727.079766:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829727.079767:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829727.079769:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829727.079770:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829727.079772:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829727.079773:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829727.079774:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829727.079776:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829727.079777:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829727.079778:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829727.079779:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829727.079781:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829727.079782:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829727.079784:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829727.079785:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829727.079787:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829727.079790:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787829727.079792:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829727.079793:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1787829727.079795:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829727.974709:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829727.974715:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829727.974917:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829727.974920:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829727.974923:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829728.998686:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829728.998692:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829728.998763:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829728.998767:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829728.998772:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829730.022692:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829730.022697:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829730.022757:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829730.022762:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829730.022766:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829731.046659:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829731.046664:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829731.046707:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829731.046711:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829731.046713:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829732.070747:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829732.070755:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829732.070815:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829732.070819:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829732.070822:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787829732.198680:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829732.198688:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1787829732.198690:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787829732.198693:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787829732.198697:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787829732.198703:0:40798:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 4532 00000100:00080000:0.0:1787829732.198711:0:40798:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787829732.198714:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829732.198715:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787829732.198718:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787829732.198720:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787829732.198722:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:0.0:1787829732.198724:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:0.0:1787829732.198726:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829732.198727:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:0.0:1787829732.198730:0:40798:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787829732.198732:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829732.198733:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787829732.198736:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787829732.198738:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829732.198739:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1787829732.198741:0:40798:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829732.198750:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787829732.198752:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829732.198755:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787829732.198757:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787829732.198760:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829732.198771:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787829732.198778:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d252c0 x1874670646859136/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787829732.198786:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787829732.198789:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829732.198802:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787829732.198804:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829732.198806:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1787829732.198808:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787829732.198809:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787829732.198811:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787829732.198812:0:40798:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4532 00000100:00080000:0.0:1787829732.198814:0:40798:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787829732.198815:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829732.198816:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787829732.198817:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787829732.198818:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787829732.198820:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000020:00000001:0.0:1787829732.198821:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:0.0:1787829732.198823:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829732.198824:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:0.0:1787829732.198828:0:40798:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787829732.198829:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829732.198830:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787829732.198834:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829732.198835:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:0.0:1787829732.198836:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829732.198840:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:3.0:1787829732.198841:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787829732.198844:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000008:00100000:0.0:1787829732.198845:0:40798:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00000001:3.0:1787829732.198848:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000008:00000001:0.0:1787829732.198849:0:40798:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829732.198852:0:40798:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829732.198852:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829732.198859:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787829732.198859:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787829732.198863:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829732.198864:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787829732.198865:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787829732.198868:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787829732.198869:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829732.198871:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829732.198875:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829732.198876:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d252c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646859136:192.168.203.143@tcp:250:kworker.0 00000100:00000001:3.0:1787829732.198881:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00100000:0.0:1787829732.198882:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d25680 x1874670646859264/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:3.0:1787829732.198885:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1787829732.198889:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829732.198893:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:0.0:1787829732.198894:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787829732.198897:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829732.198899:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829732.198900:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787829732.198905:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829732.198907:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1787829732.198908:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787829732.198909:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787829732.198911:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787829732.198912:0:40798:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4532 00000100:00080000:0.0:1787829732.198915:0:40798:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787829732.198915:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787829732.198916:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829732.198918:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601341120 : -126234108210496 : ffff8d30d4d252c0) 00000100:00000001:0.0:1787829732.198919:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787829732.198925:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787829732.198926:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:3.0:1787829732.198926:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000020:00000001:0.0:1787829732.198929:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000020:00000001:0.0:1787829732.198931:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:0.0:1787829732.198932:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829732.198933:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:0.0:1787829732.198936:0:40798:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787829732.198937:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829732.198939:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787829732.198941:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787829732.198943:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829732.198944:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1787829732.198945:0:40798:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1787829732.198947:0:40798:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829732.198948:0:40798:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829732.198952:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787829732.198953:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829732.198953:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787829732.198954:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787829732.198955:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829732.198959:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:3.0:1787829732.198961:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00100000:0.0:1787829732.198965:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d27c00 x1874670646859392/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787829732.198971:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787829732.198973:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829732.198975:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829732.198975:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00080000:0.0:1787829732.198977:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1787829732.198980:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00080000:0.0:1787829732.198981:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1787829732.198984:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:3.0:1787829732.198985:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787829732.198986:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787829732.198988:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787829732.198990:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787829732.198993:0:40798:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4532 00000100:00000001:3.0:1787829732.198994:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787829732.198997:0:40798:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787829732.198999:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4299206400 00000100:00000001:0.0:1787829732.199000:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829732.199003:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787829732.199018:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787829732.199018:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787829732.199020:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000020:00000001:0.0:1787829732.199022:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:0.0:1787829732.199023:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829732.199023:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1787829732.199026:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:0.0:1787829732.199030:0:40798:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787829732.199033:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829732.199033:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4299206400 02000000:00000001:0.0:1787829732.199037:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787829732.199041:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787829732.199042:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829732.199042:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1787829732.199047:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1787829732.199049:0:40798:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00000001:3.0:1787829732.199050:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000008:00000001:0.0:1787829732.199051:0:40798:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829732.199056:0:40798:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829732.199057:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829732.199063:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:3.0:1787829732.199063:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 02000000:00000001:0.0:1787829732.199066:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829732.199067:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787829732.199070:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00100000:3.0:1787829732.199070:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829732/real 1787829732] req@ffff8d30d4d252c0 x1874670646859136/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829757 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:0.0:1787829732.199071:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829732.199077:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787829732.199084:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d26940 x1874670646859520/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787829732.199106:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000400:00000001:3.0:1787829732.199106:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787829732.199109:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787829732.199110:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829732.199114:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829732.199115:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829732.199118:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829732.199120:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829732.199122:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829732.199123:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829732.199128:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d252c0 x1874670646859136/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829757 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829732.199133:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829732.199136:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829732.199138:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829732.199139:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829732.199140:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829732.199142:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829732.199143:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829732.199146:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829732.199149:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829732.199151:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829732.199153:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d252c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646859136:192.168.203.143@tcp:250:kworker.0 00000100:00000001:3.0:1787829732.199158:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829732.199159:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829732.199161:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829732.199162:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829732.199164:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829732.199166:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829732.199168:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829732.199171:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829732.199173:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829732.199175:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829732.199176:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829732.199178:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829732.199185:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787829732.199186:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829732.199188:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1787829732.199189:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787829732.199190:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787829732.199191:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787829732.199192:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829732.199193:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787829732.199195:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829732.199196:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829732.199199:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d25680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646859264:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829732.199201:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1787829732.199203:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1787829732.199205:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829732.199206:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1787829732.199208:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829732.199213:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787829732.199214:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601342080 : -126234108209536 : ffff8d30d4d25680) 00000100:00000001:3.0:1787829732.199218:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1787829732.199227:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:3.0:1787829732.199230:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787829732.199232:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829732.199233:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829732.199234:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1787829732.199236:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829732.199239:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4299205142 00000100:00000001:3.0:1787829732.199241:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4299206400 00000100:00000001:3.0:1787829732.199242:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4299205142 00000100:00000001:3.0:1787829732.199244:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4299206400 00000100:00000001:3.0:1787829732.199245:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829732.199246:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829732.199247:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1787829732.199250:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829732/real 1787829732] req@ffff8d30d4d25680 x1874670646859264/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829757 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1787829732.199260:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787829732.199261:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829732.199263:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829732.199264:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829732.199265:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829732.199266:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829732.199267:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829732.199269:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d25680 x1874670646859264/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829757 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829732.199274:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829732.199275:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829732.199276:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829732.199277:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829732.199278:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829732.199279:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829732.199280:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829732.199282:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829732.199284:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829732.199285:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829732.199287:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d25680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646859264:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829732.199289:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787829732.199290:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787829732.199291:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829732.199293:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787829732.199294:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829732.199295:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829732.199297:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d27c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646859392:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829732.199299:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1787829732.199300:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1787829732.199302:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829732.199303:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1787829732.199305:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829732.199307:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787829732.199308:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601351680 : -126234108199936 : ffff8d30d4d27c00) 00000100:00000001:3.0:1787829732.199311:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1787829732.199316:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:3.0:1787829732.199319:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787829732.199320:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829732.199321:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829732.199322:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1787829732.199323:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829732.199325:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4299205142 00000100:00000001:3.0:1787829732.199327:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4299206400 00000100:00000001:3.0:1787829732.199329:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4299205142 00000100:00000001:3.0:1787829732.199330:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4299206400 00000100:00000001:3.0:1787829732.199332:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829732.199333:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829732.199334:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1787829732.199336:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829732/real 1787829732] req@ffff8d30d4d27c00 x1874670646859392/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829757 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1787829732.199340:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787829732.199341:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829732.199342:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829732.199343:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829732.199344:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829732.199345:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829732.199346:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829732.199348:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d27c00 x1874670646859392/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829757 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829732.199352:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829732.199353:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829732.199364:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829732.199365:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829732.199365:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829732.199367:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829732.199368:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829732.199370:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829732.199371:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829732.199373:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829732.199374:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d27c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646859392:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829732.199377:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787829732.199378:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787829732.199379:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829732.199380:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787829732.199381:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829732.199383:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829732.199384:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d26940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646859520:192.168.203.143@tcp:8:kworker.0 00000100:00000001:3.0:1787829732.199386:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1787829732.199387:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1787829732.199388:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829732.199389:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1787829732.199391:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829732.199394:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787829732.199395:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601346880 : -126234108204736 : ffff8d30d4d26940) 00000100:00000001:3.0:1787829732.199397:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1787829732.199404:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:3.0:1787829732.199406:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787829732.199407:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829732.199408:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829732.199409:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1787829732.199411:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829732.199413:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4299205155 00000100:00000001:3.0:1787829732.199414:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4299206400 00000100:00000001:3.0:1787829732.199416:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4299205155 00000100:00000001:3.0:1787829732.199418:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4299206400 00000100:00000001:3.0:1787829732.199419:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829732.199421:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829732.199422:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1787829732.199424:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829732/real 1787829732] req@ffff8d30d4d26940 x1874670646859520/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829757 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1787829732.199429:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787829732.199429:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829732.199430:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829732.199431:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829732.199432:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829732.199433:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829732.199434:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829732.199436:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d26940 x1874670646859520/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829757 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829732.199441:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829732.199442:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829732.199443:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829732.199444:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829732.199445:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829732.199446:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829732.199447:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829732.199449:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829732.199450:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829732.199451:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829732.199453:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d26940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646859520:192.168.203.143@tcp:8:kworker.0 00000100:00000001:3.0:1787829732.199455:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829732.199456:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829732.199458:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829732.199459:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829732.199460:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829732.199461:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829732.199462:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829732.199463:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829732.199464:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829732.199466:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829732.199467:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829732.199468:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829732.199468:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829732.199469:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829732.199470:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829732.199471:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829732.199472:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829732.199473:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829732.199474:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829732.199476:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829732.199477:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829732.199478:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829732.199478:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829732.199479:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829732.199480:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829732.199480:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829732.199481:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829732.199482:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829732.199482:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829732.199484:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829732.199485:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829732.199486:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829732.199487:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787829732.199488:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829732.199489:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1787829732.199490:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829733.094659:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829733.094664:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829733.094721:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829733.094724:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829733.094727:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829734.118676:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829734.118682:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829734.118728:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829734.118731:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829734.118733:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829735.142669:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829735.142675:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829735.142714:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829735.142718:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829735.142721:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829736.166680:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829736.166687:0:815:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1787829736.166689:0:815:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1787829736.166691:0:815:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829736.166695:0:815:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1787829736.166697:0:815:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829736.166698:0:815:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829736.166700:0:815:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829736.166789:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829736.166794:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829736.166799:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829737.190670:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829737.190675:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829737.190725:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829737.190730:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829737.190734:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787829737.318745:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829737.318756:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1787829737.318758:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1787829737.318762:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787829737.318768:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1787829737.318775:0:468:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 4537 00000100:00080000:1.0:1787829737.318786:0:468:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1787829737.318789:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787829737.318791:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:1.0:1787829737.318795:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:1.0:1787829737.318797:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:1.0:1787829737.318800:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:1.0:1787829737.318802:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:1.0:1787829737.318804:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787829737.318805:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:1.0:1787829737.318808:0:468:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:1.0:1787829737.318810:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829737.318812:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1787829737.318816:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829737.318818:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829737.318820:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:1.0:1787829737.318823:0:468:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829737.318836:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1787829737.318837:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829737.318841:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1787829737.318843:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1787829737.318848:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829737.318860:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829737.318869:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d13ecf00 x1874670646859648/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787829737.318878:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787829737.318882:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829737.318897:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1787829737.318900:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829737.318903:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1787829737.318904:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1787829737.318905:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787829737.318908:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1787829737.318909:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4537 00000100:00080000:1.0:1787829737.318911:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1787829737.318913:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787829737.318914:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:1.0:1787829737.318916:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:1.0:1787829737.318918:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:1.0:1787829737.318920:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000020:00000001:1.0:1787829737.318922:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:1.0:1787829737.318923:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787829737.318924:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:1.0:1787829737.318926:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:3.0:1787829737.318927:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787829737.318928:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829737.318932:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:3.0:1787829737.318934:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 02000000:00000001:1.0:1787829737.318938:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829737.318939:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 02000000:00000001:1.0:1787829737.318944:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829737.318944:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787829737.318948:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:3.0:1787829737.318948:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000008:00100000:1.0:1787829737.318955:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 02000000:00000001:3.0:1787829737.318956:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787829737.318960:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000008:00000001:1.0:1787829737.318961:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829737.318966:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829737.318966:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829737.318988:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d13ecf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646859648:192.168.203.143@tcp:250:kworker.0 02000000:00000001:1.0:1787829737.318991:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1787829737.318994:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829737.318995:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:1.0:1787829737.318996:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1787829737.318999:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1787829737.319000:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829737.319001:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:1.0:1787829737.319005:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829737.319005:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829737.319011:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00100000:1.0:1787829737.319014:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d13eda40 x1874670646859776/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:3.0:1787829737.319018:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829737.319025:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787829737.319027:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829737.319029:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1787829737.319032:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829737.319034:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:1.0:1787829737.319036:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1787829737.319037:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787829737.319038:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787829737.319039:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:3.0:1787829737.319041:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839541341952 : -126234168209664 : ffff8d30d13ecf00) 00000100:00080000:1.0:1787829737.319043:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4537 00000100:00000001:3.0:1787829737.319048:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00080000:1.0:1787829737.319049:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1787829737.319052:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787829737.319054:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:1.0:1787829737.319056:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:1.0:1787829737.319057:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:1.0:1787829737.319058:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000020:00000001:1.0:1787829737.319060:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:1.0:1787829737.319061:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787829737.319062:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:1.0:1787829737.319064:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:1.0:1787829737.319067:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829737.319068:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1787829737.319071:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829737.319072:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829737.319074:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:1.0:1787829737.319075:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1787829737.319077:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829737.319079:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829737.319082:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1787829737.319083:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829737.319084:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1787829737.319085:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1787829737.319087:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829737.319090:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829737.319093:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d13ee1c0 x1874670646859904/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787829737.319098:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787829737.319101:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829737.319104:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:3.0:1787829737.319104:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00080000:1.0:1787829737.319109:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829737.319111:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1787829737.319112:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1787829737.319113:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787829737.319115:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1787829737.319118:0:468:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4537 00000100:00080000:1.0:1787829737.319122:0:468:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1787829737.319123:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787829737.319124:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:3.0:1787829737.319124:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000020:00000001:1.0:1787829737.319128:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:1.0:1787829737.319130:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:3.0:1787829737.319130:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000020:00000001:1.0:1787829737.319134:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:3.0:1787829737.319135:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829737.319137:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:3.0:1787829737.319138:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787829737.319141:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787829737.319143:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:1.0:1787829737.319147:0:468:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:3.0:1787829737.319147:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829737.319151:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829737.319153:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:3.0:1787829737.319155:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4299211520 02000000:00000001:1.0:1787829737.319156:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829737.319161:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829737.319162:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:1.0:1787829737.319163:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000008:00000001:1.0:1787829737.319164:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829737.319166:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829737.319169:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:3.0:1787829737.319170:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 02000000:00000001:1.0:1787829737.319171:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829737.319175:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1787829737.319176:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1787829737.319177:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829737.319180:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4299211520 00000100:00000001:1.0:1787829737.319181:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829737.319188:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d13ee580 x1874670646860032/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787829737.319189:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 00000100:00000001:3.0:1787829737.319195:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829737.319198:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787829737.319200:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829737.319202:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829737.319204:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00000001:1.0:1787829737.319205:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829737.319213:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829737/real 1787829737] req@ffff8d30d13ecf00 x1874670646859648/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829762 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1787829737.319223:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787829737.319225:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829737.319228:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829737.319231:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829737.319233:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829737.319236:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829737.319238:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829737.319241:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d13ecf00 x1874670646859648/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829762 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829737.319246:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829737.319249:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829737.319251:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829737.319252:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829737.319253:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829737.319255:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829737.319256:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829737.319259:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829737.319262:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829737.319263:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829737.319266:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d13ecf00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646859648:192.168.203.143@tcp:250:kworker.0 00000100:00000001:3.0:1787829737.319270:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829737.319272:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829737.319273:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829737.319275:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829737.319278:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829737.319279:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829737.319281:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829737.319286:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829737.319287:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829737.319291:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829737.319292:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829737.319294:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829737.319301:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787829737.319303:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829737.319304:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1787829737.319305:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787829737.319306:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787829737.319307:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787829737.319308:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829737.319310:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787829737.319312:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829737.319314:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829737.319316:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d13eda40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646859776:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829737.319318:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1787829737.319320:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1787829737.319322:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829737.319333:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1787829737.319336:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829737.319344:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787829737.319345:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839541344832 : -126234168206784 : ffff8d30d13eda40) 00000100:00000001:3.0:1787829737.319351:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1787829737.319365:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:3.0:1787829737.319371:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787829737.319374:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829737.319375:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829737.319377:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1787829737.319380:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829737.319385:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4299210262 00000100:00000001:3.0:1787829737.319389:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4299211520 00000100:00000001:3.0:1787829737.319391:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4299210262 00000100:00000001:3.0:1787829737.319394:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4299211520 00000100:00000001:3.0:1787829737.319396:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829737.319399:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829737.319401:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1787829737.319406:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829737/real 1787829737] req@ffff8d30d13eda40 x1874670646859776/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829762 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1787829737.319416:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787829737.319418:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829737.319420:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829737.319423:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829737.319424:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829737.319425:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829737.319427:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829737.319430:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d13eda40 x1874670646859776/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829762 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829737.319435:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829737.319437:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829737.319439:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829737.319439:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829737.319440:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829737.319441:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829737.319443:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829737.319444:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829737.319446:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829737.319447:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829737.319450:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d13eda40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646859776:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829737.319452:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787829737.319453:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787829737.319455:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829737.319456:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787829737.319458:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829737.319459:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829737.319461:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d13ee1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646859904:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829737.319463:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1787829737.319464:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1787829737.319466:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829737.319467:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1787829737.319469:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829737.319473:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787829737.319474:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839541346752 : -126234168204864 : ffff8d30d13ee1c0) 00000100:00000001:3.0:1787829737.319477:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1787829737.319486:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:3.0:1787829737.319489:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787829737.319491:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829737.319492:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829737.319494:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1787829737.319496:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829737.319498:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4299210262 00000100:00000001:3.0:1787829737.319500:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4299211520 00000100:00000001:3.0:1787829737.319501:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4299210262 00000100:00000001:3.0:1787829737.319503:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4299211520 00000100:00000001:3.0:1787829737.319505:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829737.319506:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829737.319507:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1787829737.319509:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829737/real 1787829737] req@ffff8d30d13ee1c0 x1874670646859904/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829762 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1787829737.319514:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787829737.319516:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829737.319525:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829737.319527:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829737.319529:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829737.319531:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829737.319532:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829737.319536:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d13ee1c0 x1874670646859904/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829762 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829737.319544:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829737.319546:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829737.319549:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829737.319551:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829737.319552:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829737.319554:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829737.319556:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829737.319559:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829737.319562:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829737.319565:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829737.319568:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d13ee1c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646859904:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829737.319571:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787829737.319572:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787829737.319573:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829737.319575:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787829737.319576:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829737.319577:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829737.319579:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d13ee580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646860032:192.168.203.143@tcp:8:kworker.0 00000100:00000001:3.0:1787829737.319582:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1787829737.319583:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1787829737.319584:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829737.319585:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1787829737.319588:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829737.319593:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787829737.319594:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839541347712 : -126234168203904 : ffff8d30d13ee580) 00000100:00000001:3.0:1787829737.319596:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1787829737.319628:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:3.0:1787829737.319632:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787829737.319634:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829737.319635:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829737.319636:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1787829737.319638:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829737.319640:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4299210276 00000100:00000001:3.0:1787829737.319642:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4299211520 00000100:00000001:3.0:1787829737.319644:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4299210276 00000100:00000001:3.0:1787829737.319645:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4299211520 00000100:00000001:3.0:1787829737.319648:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829737.319650:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829737.319652:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1787829737.319656:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829737/real 1787829737] req@ffff8d30d13ee580 x1874670646860032/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829762 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1787829737.319672:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787829737.319673:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829737.319676:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829737.319678:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829737.319680:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829737.319681:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829737.319683:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829737.319688:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d13ee580 x1874670646860032/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829762 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829737.319696:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829737.319698:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829737.319701:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829737.319702:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829737.319703:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829737.319705:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829737.319706:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829737.319708:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829737.319711:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829737.319714:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829737.319717:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d13ee580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646860032:192.168.203.143@tcp:8:kworker.0 00000100:00000001:3.0:1787829737.319722:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829737.319724:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829737.319726:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829737.319727:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829737.319730:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829737.319731:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829737.319733:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829737.319737:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829737.319738:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829737.319741:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829737.319742:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829737.319745:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829737.319746:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829737.319748:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829737.319750:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829737.319751:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829737.319752:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829737.319754:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829737.319755:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829737.319758:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829737.319759:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829737.319760:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829737.319761:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829737.319762:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829737.319763:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829737.319764:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829737.319765:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829737.319766:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829737.319766:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829737.319767:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829737.319768:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829737.319769:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829737.319772:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787829737.319772:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829737.319774:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1787829737.319775:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829738.214665:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829738.214675:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829738.214732:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829738.214736:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829738.214739:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829739.238737:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829739.238744:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829739.238810:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829739.238814:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829739.238817:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829740.262667:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829740.262673:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829740.262727:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829740.262730:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829740.262733:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829741.286698:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829741.286703:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829741.286758:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829741.286762:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829741.286766:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829742.310713:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829742.310722:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829742.310804:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829742.310809:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829742.310815:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787829742.438732:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829742.438747:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1787829742.438751:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787829742.438756:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787829742.438761:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787829742.438771:0:468:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 4542 00000100:00080000:3.0:1787829742.438786:0:468:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787829742.438790:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829742.438791:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:3.0:1787829742.438795:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1787829742.438800:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1787829742.438803:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:3.0:1787829742.438805:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:3.0:1787829742.438806:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829742.438807:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:3.0:1787829742.438811:0:468:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:3.0:1787829742.438813:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829742.438816:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787829742.438820:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829742.438822:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829742.438824:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:3.0:1787829742.438827:0:468:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829742.438840:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787829742.438841:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829742.438845:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787829742.438848:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787829742.438853:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829742.438866:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829742.438875:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c7016d00 x1874670646860160/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787829742.438884:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787829742.438888:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829742.438905:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787829742.438907:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829742.438910:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1787829742.438911:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787829742.438912:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787829742.438915:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787829742.438917:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4542 00000100:00080000:3.0:1787829742.438920:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787829742.438921:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829742.438923:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:3.0:1787829742.438926:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1787829742.438927:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1787829742.438929:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000020:00000001:3.0:1787829742.438931:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:3.0:1787829742.438933:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829742.438949:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:1.0:1787829742.438952:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00080000:3.0:1787829742.438953:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:3.0:1787829742.438958:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829742.438960:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1787829742.438961:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 02000000:00000001:3.0:1787829742.438965:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829742.438966:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 02000000:00000001:3.0:1787829742.438970:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829742.438971:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787829742.438973:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:1.0:1787829742.438976:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000008:00100000:3.0:1787829742.438979:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1787829742.438982:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829742.438983:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:3.0:1787829742.438985:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829742.438989:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829742.438990:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787829742.438992:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829742.438996:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829742.438997:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787829742.439000:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787829742.439003:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829742.439005:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c7016d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646860160:192.168.203.143@tcp:250:kworker.0 00000100:00000001:3.0:1787829742.439009:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829742.439012:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00100000:3.0:1787829742.439016:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c70170c0 x1874670646860288/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:1.0:1787829742.439019:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787829742.439023:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829742.439026:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 02000000:00000001:1.0:1787829742.439027:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:3.0:1787829742.439030:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829742.439034:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829742.439035:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787829742.439040:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829742.439059:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1787829742.439061:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787829742.439063:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787829742.439065:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787829742.439067:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4542 00000100:00080000:3.0:1787829742.439071:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1787829742.439072:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787829742.439073:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787829742.439075:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369547008 : -126234340004608 : ffff8d30c7016d00) 00000100:00000001:3.0:1787829742.439076:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:3.0:1787829742.439080:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:1.0:1787829742.439081:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000020:00000001:3.0:1787829742.439082:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1787829742.439085:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000020:00000001:3.0:1787829742.439087:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:3.0:1787829742.439089:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829742.439090:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:3.0:1787829742.439092:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:3.0:1787829742.439093:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829742.439095:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787829742.439097:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829742.439098:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829742.439099:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:3.0:1787829742.439101:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1787829742.439103:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829742.439104:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829742.439108:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787829742.439109:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829742.439111:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787829742.439111:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787829742.439113:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829742.439116:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829742.439136:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c7017840 x1874670646860416/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787829742.439144:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787829742.439146:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829742.439149:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:1.0:1787829742.439153:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00080000:3.0:1787829742.439154:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829742.439160:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1787829742.439163:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787829742.439165:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787829742.439167:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787829742.439169:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:1.0:1787829742.439173:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00080000:3.0:1787829742.439176:0:468:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4542 00000100:00000001:1.0:1787829742.439177:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829742.439181:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00080000:3.0:1787829742.439186:0:468:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1787829742.439187:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829742.439191:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829742.439193:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:1.0:1787829742.439194:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4299216640 00000020:00000001:3.0:1787829742.439195:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1787829742.439198:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1787829742.439201:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000020:00000001:3.0:1787829742.439203:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:3.0:1787829742.439204:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829742.439205:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:1.0:1787829742.439206:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 00000100:00080000:3.0:1787829742.439208:0:468:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:3.0:1787829742.439212:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829742.439213:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787829742.439216:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829742.439218:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4299216640 02000000:00000001:3.0:1787829742.439218:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829742.439222:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:3.0:1787829742.439224:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1787829742.439226:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829742.439228:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 00000100:00000001:3.0:1787829742.439228:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829742.439236:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829742.439236:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787829742.439239:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829742.439241:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:1.0:1787829742.439242:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829742.439243:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787829742.439246:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829742.439250:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00000001:3.0:1787829742.439252:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829742.439258:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c7014000 x1874670646860544/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787829742.439261:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829742/real 1787829742] req@ffff8d30c7016d00 x1874670646860160/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829767 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829742.439277:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787829742.439282:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:1.0:1787829742.439283:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787829742.439286:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829742.439286:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829742.439293:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787829742.439296:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787829742.439298:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829742.439300:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829742.439302:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787829742.439306:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c7016d00 x1874670646860160/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829767 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787829742.439331:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829742.439335:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787829742.439338:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829742.439339:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829742.439340:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829742.439342:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787829742.439344:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787829742.439346:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829742.439350:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787829742.439351:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787829742.439355:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c7016d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646860160:192.168.203.143@tcp:250:kworker.0 00000100:00000001:1.0:1787829742.439359:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829742.439361:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829742.439363:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787829742.439366:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787829742.439368:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787829742.439370:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787829742.439372:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787829742.439376:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787829742.439378:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787829742.439380:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787829742.439382:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829742.439384:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829742.439399:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787829742.439401:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829742.439403:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787829742.439404:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1787829742.439405:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787829742.439407:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787829742.439408:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829742.439410:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787829742.439412:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829742.439414:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829742.439416:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c70170c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646860288:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787829742.439419:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787829742.439421:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787829742.439422:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829742.439423:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787829742.439425:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829742.439431:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787829742.439449:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369547968 : -126234340003648 : ffff8d30c70170c0) 00000100:00000001:1.0:1787829742.439453:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1787829742.439464:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787829742.439467:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787829742.439469:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829742.439470:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829742.439471:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787829742.439473:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829742.439477:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4299215382 00000100:00000001:1.0:1787829742.439479:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4299216640 00000100:00000001:1.0:1787829742.439481:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4299215382 00000100:00000001:1.0:1787829742.439482:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4299216640 00000100:00000001:1.0:1787829742.439484:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829742.439485:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829742.439486:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787829742.439489:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829742/real 1787829742] req@ffff8d30c70170c0 x1874670646860288/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829767 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787829742.439494:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787829742.439495:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787829742.439497:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787829742.439498:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787829742.439499:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829742.439501:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829742.439501:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787829742.439504:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c70170c0 x1874670646860288/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829767 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787829742.439508:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829742.439510:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787829742.439511:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829742.439512:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829742.439513:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829742.439514:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787829742.439515:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787829742.439517:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829742.439519:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787829742.439520:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787829742.439522:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c70170c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646860288:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787829742.439524:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787829742.439525:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787829742.439526:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829742.439528:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787829742.439529:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829742.439530:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829742.439532:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c7017840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646860416:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787829742.439534:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787829742.439535:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787829742.439536:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829742.439537:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787829742.439539:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829742.439542:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787829742.439543:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369549888 : -126234340001728 : ffff8d30c7017840) 00000100:00000001:1.0:1787829742.439545:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1787829742.439552:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787829742.439554:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787829742.439555:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829742.439556:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829742.439557:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787829742.439559:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829742.439561:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4299215382 00000100:00000001:1.0:1787829742.439563:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4299216640 00000100:00000001:1.0:1787829742.439564:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4299215382 00000100:00000001:1.0:1787829742.439566:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4299216640 00000100:00000001:1.0:1787829742.439567:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829742.439568:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829742.439569:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787829742.439572:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829742/real 1787829742] req@ffff8d30c7017840 x1874670646860416/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829767 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787829742.439576:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787829742.439577:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787829742.439578:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787829742.439579:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787829742.439580:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829742.439581:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829742.439582:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787829742.439584:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c7017840 x1874670646860416/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829767 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787829742.439588:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829742.439590:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787829742.439591:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829742.439592:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829742.439593:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829742.439594:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787829742.439595:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787829742.439597:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829742.439599:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787829742.439600:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787829742.439602:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c7017840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646860416:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787829742.439605:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787829742.439606:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787829742.439606:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829742.439608:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787829742.439609:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829742.439610:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829742.439634:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c7014000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646860544:192.168.203.143@tcp:8:kworker.0 00000100:00000001:1.0:1787829742.439637:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787829742.439638:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787829742.439639:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829742.439640:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787829742.439642:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829742.439646:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787829742.439646:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369535488 : -126234340016128 : ffff8d30c7014000) 00000100:00000001:1.0:1787829742.439649:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1787829742.439655:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787829742.439658:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787829742.439660:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829742.439661:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829742.439662:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787829742.439664:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829742.439666:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4299215396 00000100:00000001:1.0:1787829742.439667:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4299216640 00000100:00000001:1.0:1787829742.439669:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4299215396 00000100:00000001:1.0:1787829742.439670:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4299216640 00000100:00000001:1.0:1787829742.439672:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829742.439673:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829742.439674:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787829742.439677:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829742/real 1787829742] req@ffff8d30c7014000 x1874670646860544/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829767 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787829742.439682:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787829742.439683:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787829742.439685:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787829742.439686:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787829742.439687:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829742.439688:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829742.439689:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787829742.439691:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c7014000 x1874670646860544/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829767 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787829742.439695:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829742.439697:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787829742.439698:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829742.439699:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829742.439700:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829742.439701:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787829742.439702:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787829742.439703:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829742.439705:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787829742.439706:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787829742.439708:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c7014000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646860544:192.168.203.143@tcp:8:kworker.0 00000100:00000001:1.0:1787829742.439711:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829742.439712:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829742.439713:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787829742.439714:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787829742.439715:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787829742.439716:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787829742.439717:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787829742.439719:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787829742.439720:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787829742.439721:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787829742.439722:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829742.439723:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829742.439724:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787829742.439725:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787829742.439728:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787829742.439729:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787829742.439730:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787829742.439731:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787829742.439731:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787829742.439732:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787829742.439733:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829742.439734:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829742.439735:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787829742.439735:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787829742.439736:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787829742.439737:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787829742.439737:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787829742.439738:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787829742.439739:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787829742.439740:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787829742.439741:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829742.439742:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829742.439744:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787829742.439745:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829742.439746:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787829742.439747:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829743.334735:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829743.334741:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829743.334799:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829743.334804:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829743.334808:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829744.358697:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829744.358703:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829744.358780:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829744.358783:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829744.358786:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829745.382711:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829745.382728:0:815:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1787829745.382730:0:815:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1787829745.382734:0:815:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829745.382740:0:815:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1787829745.382742:0:815:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829745.382744:0:815:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829745.382746:0:815:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829745.382787:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829745.382790:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829745.382793:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829746.406680:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829746.406686:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829746.406729:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829746.406733:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829746.406736:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829747.430668:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829747.430673:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829747.430718:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829747.430721:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829747.430724:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787829747.558732:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829747.558741:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1787829747.558743:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787829747.558745:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787829747.558749:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787829747.558755:0:40798:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 4548 00000100:00080000:0.0:1787829747.558764:0:40798:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787829747.558766:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829747.558768:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787829747.558771:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787829747.558773:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787829747.558776:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:0.0:1787829747.558778:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:0.0:1787829747.558780:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829747.558781:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:0.0:1787829747.558784:0:40798:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787829747.558786:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829747.558788:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787829747.558790:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787829747.558792:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829747.558794:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1787829747.558796:0:40798:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829747.558805:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787829747.558807:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829747.558810:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787829747.558812:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787829747.558815:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829747.558826:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787829747.558833:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d26940 x1874670646860672/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787829747.558841:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787829747.558844:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829747.558855:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787829747.558857:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829747.558859:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1787829747.558861:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787829747.558862:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787829747.558864:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787829747.558865:0:40798:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4548 00000100:00080000:0.0:1787829747.558867:0:40798:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787829747.558869:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829747.558870:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787829747.558871:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787829747.558873:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787829747.558875:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000020:00000001:0.0:1787829747.558877:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:0.0:1787829747.558878:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829747.558880:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:0.0:1787829747.558882:0:40798:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787829747.558884:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829747.558885:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787829747.558890:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829747.558891:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:0.0:1787829747.558892:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829747.558895:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:1.0:1787829747.558896:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000008:00100000:0.0:1787829747.558912:0:40798:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00000001:1.0:1787829747.558913:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787829747.558916:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000008:00000001:0.0:1787829747.558919:0:40798:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829747.558920:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829747.558925:0:40798:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829747.558927:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787829747.558930:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787829747.558932:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1787829747.558933:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829747.558935:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829747.558939:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d26940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646860672:192.168.203.143@tcp:250:kworker.0 00000100:00000001:0.0:1787829747.558941:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787829747.558944:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:1.0:1787829747.558948:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787829747.558949:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829747.558957:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829747.558957:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787829747.558962:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787829747.558965:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d27c00 x1874670646860800/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:1.0:1787829747.558968:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:0.0:1787829747.558975:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 02000000:00000001:1.0:1787829747.558976:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829747.558977:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829747.558983:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787829747.558985:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829747.558988:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1787829747.558989:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787829747.558990:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787829747.558992:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787829747.558994:0:40798:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4548 00000100:00080000:0.0:1787829747.558997:0:40798:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1787829747.558997:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787829747.559000:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601346880 : -126234108204736 : ffff8d30d4d26940) 00000100:00000001:0.0:1787829747.559002:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829747.559004:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787829747.559007:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:1.0:1787829747.559007:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000020:00000001:0.0:1787829747.559011:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787829747.559012:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000020:00000001:0.0:1787829747.559014:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:0.0:1787829747.559015:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829747.559016:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:0.0:1787829747.559019:0:40798:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787829747.559021:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829747.559022:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787829747.559025:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787829747.559026:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829747.559027:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1787829747.559029:0:40798:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1787829747.559030:0:40798:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829747.559032:0:40798:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829747.559035:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787829747.559036:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829747.559037:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787829747.559038:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787829747.559039:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829747.559043:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:1.0:1787829747.559048:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00100000:0.0:1787829747.559050:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d25680 x1874670646860928/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787829747.559057:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787829747.559059:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829747.559060:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787829747.559063:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1787829747.559064:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00080000:0.0:1787829747.559065:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1787829747.559068:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787829747.559069:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787829747.559071:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787829747.559072:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:1.0:1787829747.559076:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787829747.559077:0:40798:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4548 00000100:00080000:0.0:1787829747.559080:0:40798:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1787829747.559081:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787829747.559085:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829747.559087:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787829747.559088:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:1.0:1787829747.559089:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787829747.559090:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787829747.559092:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000020:00000001:0.0:1787829747.559096:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:1.0:1787829747.559096:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4299221760 00000100:00000001:0.0:1787829747.559102:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829747.559104:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:1.0:1787829747.559106:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 00000100:00080000:0.0:1787829747.559110:0:40798:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:1.0:1787829747.559113:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4299221760 00000100:00000001:0.0:1787829747.559115:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829747.559118:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 02000000:00000001:0.0:1787829747.559120:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1787829747.559122:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829747.559125:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829747.559126:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829747.559128:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 02000000:00000001:0.0:1787829747.559132:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829747.559135:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829747/real 1787829747] req@ffff8d30d4d26940 x1874670646860672/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829772 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787829747.559136:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1787829747.559141:0:40798:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1787829747.559149:0:40798:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829747.559151:0:40798:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:1.0:1787829747.559151:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787829747.559154:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 02000000:00000001:0.0:1787829747.559158:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:1.0:1787829747.559159:0:2390:0:(events.c:78:reply_in_callback()) Process entered 02000000:00000001:0.0:1787829747.559161:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829747.559163:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787829747.559165:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:1.0:1787829747.559166:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:0.0:1787829747.559167:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:1.0:1787829747.559169:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 02000000:00000001:0.0:1787829747.559171:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829747.559172:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829747.559177:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829747.559178:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d26940 x1874670646860672/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829772 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829747.559187:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d252c0 x1874670646861056/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787829747.559190:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829747.559195:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787829747.559199:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787829747.559200:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829747.559204:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829747.559204:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829747.559208:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829747.559209:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829747.559213:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787829747.559218:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787829747.559221:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829747.559228:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787829747.559231:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787829747.559236:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d26940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646860672:192.168.203.143@tcp:250:kworker.0 00000100:00000001:1.0:1787829747.559241:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829747.559243:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829747.559245:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787829747.559247:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787829747.559252:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787829747.559254:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787829747.559256:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787829747.559260:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787829747.559261:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787829747.559264:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787829747.559265:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829747.559266:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829747.559272:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787829747.559274:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829747.559275:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787829747.559276:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1787829747.559278:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787829747.559279:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787829747.559280:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829747.559282:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787829747.559283:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829747.559285:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829747.559288:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d27c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646860800:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787829747.559290:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787829747.559292:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787829747.559293:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829747.559295:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787829747.559298:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829747.559304:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787829747.559306:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601351680 : -126234108199936 : ffff8d30d4d27c00) 00000100:00000001:1.0:1787829747.559310:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1787829747.559324:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787829747.559328:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787829747.559331:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829747.559333:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829747.559334:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787829747.559337:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829747.559342:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4299220502 00000100:00000001:1.0:1787829747.559345:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4299221760 00000100:00000001:1.0:1787829747.559347:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4299220502 00000100:00000001:1.0:1787829747.559350:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4299221760 00000100:00000001:1.0:1787829747.559352:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829747.559354:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829747.559356:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787829747.559361:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829747/real 1787829747] req@ffff8d30d4d27c00 x1874670646860800/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829772 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787829747.559371:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787829747.559372:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787829747.559375:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787829747.559377:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787829747.559378:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829747.559380:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829747.559382:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787829747.559386:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d27c00 x1874670646860800/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829772 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787829747.559397:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829747.559398:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787829747.559400:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829747.559401:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829747.559402:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829747.559403:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787829747.559404:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787829747.559406:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829747.559408:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787829747.559409:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787829747.559411:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d27c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646860800:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787829747.559414:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787829747.559415:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787829747.559416:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829747.559417:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787829747.559419:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829747.559420:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829747.559422:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d25680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646860928:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787829747.559424:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787829747.559426:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787829747.559427:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829747.559428:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787829747.559430:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829747.559434:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787829747.559435:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601342080 : -126234108209536 : ffff8d30d4d25680) 00000100:00000001:1.0:1787829747.559438:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1787829747.559447:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787829747.559450:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787829747.559451:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829747.559452:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829747.559453:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787829747.559455:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829747.559458:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4299220502 00000100:00000001:1.0:1787829747.559459:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4299221760 00000100:00000001:1.0:1787829747.559461:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4299220502 00000100:00000001:1.0:1787829747.559462:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4299221760 00000100:00000001:1.0:1787829747.559464:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829747.559465:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829747.559466:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787829747.559468:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829747/real 1787829747] req@ffff8d30d4d25680 x1874670646860928/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829772 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787829747.559476:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787829747.559477:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787829747.559479:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787829747.559480:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787829747.559482:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829747.559483:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829747.559485:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787829747.559489:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d25680 x1874670646860928/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829772 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787829747.559496:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829747.559498:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787829747.559501:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829747.559503:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829747.559504:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829747.559506:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787829747.559508:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787829747.559511:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829747.559514:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787829747.559516:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787829747.559521:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d25680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646860928:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787829747.559524:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787829747.559525:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787829747.559526:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829747.559528:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787829747.559529:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829747.559531:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829747.559533:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d252c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646861056:192.168.203.143@tcp:8:kworker.0 00000100:00000001:1.0:1787829747.559535:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787829747.559536:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787829747.559537:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829747.559538:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787829747.559540:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829747.559545:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787829747.559546:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601341120 : -126234108210496 : ffff8d30d4d252c0) 00000100:00000001:1.0:1787829747.559549:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1787829747.559557:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787829747.559560:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787829747.559562:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829747.559563:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829747.559564:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787829747.559565:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829747.559568:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4299220515 00000100:00000001:1.0:1787829747.559570:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4299221760 00000100:00000001:1.0:1787829747.559571:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4299220515 00000100:00000001:1.0:1787829747.559572:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4299221760 00000100:00000001:1.0:1787829747.559574:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829747.559576:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829747.559578:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787829747.559582:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829747/real 1787829747] req@ffff8d30d4d252c0 x1874670646861056/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829772 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787829747.559596:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787829747.559598:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787829747.559628:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787829747.559630:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787829747.559632:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829747.559633:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829747.559634:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787829747.559638:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d252c0 x1874670646861056/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829772 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787829747.559643:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829747.559645:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787829747.559646:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829747.559647:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829747.559648:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829747.559649:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787829747.559651:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787829747.559652:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829747.559654:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787829747.559655:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787829747.559657:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d252c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646861056:192.168.203.143@tcp:8:kworker.0 00000100:00000001:1.0:1787829747.559660:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829747.559662:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829747.559663:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787829747.559664:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787829747.559666:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787829747.559667:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787829747.559667:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787829747.559669:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787829747.559670:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787829747.559672:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787829747.559673:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829747.559675:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829747.559676:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787829747.559676:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787829747.559678:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787829747.559679:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787829747.559679:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787829747.559681:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787829747.559681:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787829747.559683:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787829747.559683:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829747.559684:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829747.559685:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787829747.559686:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787829747.559687:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787829747.559688:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787829747.559688:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787829747.559689:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787829747.559690:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787829747.559691:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787829747.559692:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829747.559693:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829747.559695:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787829747.559696:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829747.559697:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787829747.559698:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829748.454668:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829748.454673:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829748.454726:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829748.454729:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829748.454732:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829749.478720:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829749.478727:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829749.478814:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829749.478819:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829749.478823:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829750.502669:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829750.502676:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829750.502749:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829750.502752:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829750.502755:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829751.526686:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829751.526691:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829751.526739:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829751.526742:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829751.526745:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829752.550682:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829752.550688:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829752.550731:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829752.550734:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829752.550737:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787829752.678708:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829752.678719:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1787829752.678722:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787829752.678727:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787829752.678732:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787829752.678741:0:40798:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 4553 00000100:00080000:0.0:1787829752.678751:0:40798:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787829752.678753:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829752.678755:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787829752.678758:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787829752.678761:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787829752.678763:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:0.0:1787829752.678765:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:0.0:1787829752.678767:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829752.678768:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:0.0:1787829752.678772:0:40798:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787829752.678774:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829752.678776:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787829752.678779:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787829752.678781:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829752.678783:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1787829752.678784:0:40798:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829752.678794:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787829752.678795:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829752.678798:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787829752.678800:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787829752.678804:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829752.678813:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787829752.678820:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d25a40 x1874670646861184/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787829752.678828:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787829752.678831:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829752.678842:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787829752.678845:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829752.678847:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1787829752.678848:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787829752.678850:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787829752.678852:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787829752.678854:0:40798:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4553 00000100:00080000:0.0:1787829752.678857:0:40798:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787829752.678859:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787829752.678860:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787829752.678861:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787829752.678877:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:1.0:1787829752.678878:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000020:00000001:0.0:1787829752.678879:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:1.0:1787829752.678882:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000020:00000001:0.0:1787829752.678883:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:1.0:1787829752.678885:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000020:00000001:0.0:1787829752.678886:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:1.0:1787829752.678888:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829752.678890:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829752.678892:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 02000000:00000001:1.0:1787829752.678893:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000100:00080000:0.0:1787829752.678897:0:40798:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 02000000:00000001:1.0:1787829752.678897:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787829752.678903:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829752.678903:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829752.678908:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00100000:1.0:1787829752.678912:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d25a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646861184:192.168.203.143@tcp:250:kworker.0 02000000:00000001:0.0:1787829752.678914:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829752.678918:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787829752.678919:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829752.678922:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 02000000:00000001:1.0:1787829752.678923:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000008:00100000:0.0:1787829752.678926:0:40798:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 02000000:00000001:1.0:1787829752.678927:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:0.0:1787829752.678932:0:40798:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829752.678933:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:0.0:1787829752.678937:0:40798:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829752.678939:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829752.678941:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787829752.678943:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829752.678944:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787829752.678945:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787829752.678947:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829752.678952:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829752.678954:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00100000:0.0:1787829752.678958:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d24f00 x1874670646861312/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787829752.678958:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601343040 : -126234108208576 : ffff8d30d4d25a40) 00000100:00000001:1.0:1787829752.678962:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:0.0:1787829752.678972:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787829752.678974:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829752.678977:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787829752.678980:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829752.678982:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1787829752.678983:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787829752.678985:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787829752.678987:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787829752.678989:0:40798:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4553 00000100:00080000:0.0:1787829752.678991:0:40798:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787829752.678993:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829752.678996:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000800:00000100:1.0:1787829752.678996:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000020:00000001:0.0:1787829752.678998:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787829752.679000:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787829752.679002:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000020:00000001:0.0:1787829752.679004:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:0.0:1787829752.679006:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787829752.679007:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1787829752.679008:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:1.0:1787829752.679012:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00080000:0.0:1787829752.679013:0:40798:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:1.0:1787829752.679015:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829752.679016:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829752.679017:0:2390:0:(events.c:68:request_out_callback()) Process leaving 02000000:00000001:0.0:1787829752.679019:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787829752.679022:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829752.679023:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829752.679024:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829752.679028:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:1.0:1787829752.679029:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4299226880 00000008:00100000:0.0:1787829752.679030:0:40798:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1787829752.679034:0:40798:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829752.679036:0:40798:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829752.679039:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787829752.679040:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829752.679040:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1787829752.679044:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787829752.679045:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787829752.679046:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829752.679048:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4299226880 00000100:00000001:0.0:1787829752.679051:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829752.679053:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 00000100:00100000:0.0:1787829752.679057:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d24000 x1874670646861440/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787829752.679057:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829752.679063:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829752.679068:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00000001:0.0:1787829752.679071:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787829752.679075:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829752.679075:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829752/real 1787829752] req@ffff8d30d4d25a40 x1874670646861184/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829777 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787829752.679081:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787829752.679088:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000400:00000001:1.0:1787829752.679090:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00080000:0.0:1787829752.679093:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000400:00000001:1.0:1787829752.679093:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787829752.679097:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787829752.679098:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:1.0:1787829752.679100:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00080000:0.0:1787829752.679102:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787829752.679103:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829752.679106:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787829752.679108:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:1.0:1787829752.679109:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:0.0:1787829752.679118:0:40798:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4553 00000100:00100000:1.0:1787829752.679118:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d25a40 x1874670646861184/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829777 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829752.679127:0:40798:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787829752.679133:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829752.679135:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:1.0:1787829752.679137:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1787829752.679139:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787829752.679141:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:1.0:1787829752.679142:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000020:00000001:0.0:1787829752.679145:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:1.0:1787829752.679146:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1787829752.679151:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:1.0:1787829752.679152:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829752.679154:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829752.679157:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787829752.679157:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787829752.679160:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:1.0:1787829752.679161:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787829752.679166:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829752.679168:0:40798:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00080000:1.0:1787829752.679172:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787829752.679174:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787829752.679176:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d25a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646861184:192.168.203.143@tcp:250:kworker.0 00000100:00000001:0.0:1787829752.679180:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829752.679180:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829752.679182:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 02000000:00000001:0.0:1787829752.679183:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1787829752.679184:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787829752.679188:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829752.679188:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787829752.679192:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 02000000:00000001:0.0:1787829752.679193:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829752.679195:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000100:00000001:0.0:1787829752.679196:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 00000020:00000001:1.0:1787829752.679199:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 00000008:00100000:0.0:1787829752.679200:0:40798:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1787829752.679204:0:40798:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829752.679204:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 00000100:00000001:0.0:1787829752.679207:0:40798:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829752.679208:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787829752.679212:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 02000000:00000001:0.0:1787829752.679213:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:1.0:1787829752.679214:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:0.0:1787829752.679216:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829752.679218:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829752.679221:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787829752.679223:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787829752.679224:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829752.679226:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787829752.679228:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829752.679229:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829752.679231:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787829752.679234:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00100000:0.0:1787829752.679237:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d270c0 x1874670646861568/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787829752.679237:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787829752.679240:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787829752.679242:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829752.679247:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787829752.679249:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787829752.679250:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787829752.679254:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829752.679254:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829752.679261:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829752.679262:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d24f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646861312:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787829752.679267:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787829752.679268:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787829752.679269:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829752.679271:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787829752.679273:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829752.679278:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787829752.679279:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601340160 : -126234108211456 : ffff8d30d4d24f00) 00000100:00000001:1.0:1787829752.679282:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1787829752.679291:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787829752.679294:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787829752.679296:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829752.679297:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829752.679298:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787829752.679300:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829752.679303:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4299225622 00000100:00000001:1.0:1787829752.679305:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4299226880 00000100:00000001:1.0:1787829752.679314:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4299225622 00000100:00000001:1.0:1787829752.679316:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4299226880 00000100:00000001:1.0:1787829752.679317:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829752.679319:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829752.679320:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787829752.679323:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829752/real 1787829752] req@ffff8d30d4d24f00 x1874670646861312/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829777 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787829752.679329:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787829752.679330:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787829752.679332:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787829752.679333:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787829752.679334:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829752.679335:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829752.679336:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787829752.679339:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d24f00 x1874670646861312/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829777 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787829752.679348:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829752.679349:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787829752.679351:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829752.679352:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829752.679353:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829752.679354:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787829752.679355:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787829752.679357:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829752.679359:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787829752.679360:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787829752.679362:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d24f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646861312:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787829752.679364:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787829752.679365:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787829752.679366:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829752.679368:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787829752.679369:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829752.679370:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829752.679372:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d24000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646861440:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787829752.679374:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787829752.679376:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787829752.679377:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829752.679378:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787829752.679380:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829752.679382:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787829752.679383:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601336320 : -126234108215296 : ffff8d30d4d24000) 00000100:00000001:1.0:1787829752.679385:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1787829752.679392:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787829752.679396:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787829752.679397:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829752.679398:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829752.679400:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787829752.679401:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829752.679404:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4299225622 00000100:00000001:1.0:1787829752.679406:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4299226880 00000100:00000001:1.0:1787829752.679408:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4299225622 00000100:00000001:1.0:1787829752.679409:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4299226880 00000100:00000001:1.0:1787829752.679411:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829752.679412:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829752.679413:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787829752.679416:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829752/real 1787829752] req@ffff8d30d4d24000 x1874670646861440/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829777 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787829752.679421:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787829752.679421:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787829752.679423:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787829752.679424:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787829752.679424:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829752.679426:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829752.679427:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787829752.679429:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d24000 x1874670646861440/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829777 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787829752.679433:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829752.679435:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787829752.679436:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829752.679437:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829752.679438:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829752.679439:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787829752.679440:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787829752.679441:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829752.679443:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787829752.679444:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787829752.679447:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d24000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646861440:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787829752.679449:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829752.679450:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829752.679451:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787829752.679452:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787829752.679453:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787829752.679454:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787829752.679455:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787829752.679456:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787829752.679457:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787829752.679459:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787829752.679460:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829752.679461:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829752.679462:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787829752.679463:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787829752.679465:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787829752.679466:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787829752.679466:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787829752.679468:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787829752.679468:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787829752.679469:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787829752.679470:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829752.679471:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829752.679473:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787829752.679474:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829752.679475:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787829752.679476:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1787829752.679477:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787829752.679478:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787829752.679479:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829752.679481:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787829752.679482:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829752.679483:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829752.679485:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d270c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646861568:192.168.203.143@tcp:8:kworker.0 00000100:00000001:1.0:1787829752.679487:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787829752.679488:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787829752.679489:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829752.679490:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787829752.679492:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829752.679495:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787829752.679496:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601348800 : -126234108202816 : ffff8d30d4d270c0) 00000100:00000001:1.0:1787829752.679499:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1787829752.679509:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787829752.679513:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787829752.679516:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829752.679517:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829752.679519:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787829752.679521:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829752.679525:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4299225635 00000100:00000001:1.0:1787829752.679528:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4299226880 00000100:00000001:1.0:1787829752.679530:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4299225635 00000100:00000001:1.0:1787829752.679533:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4299226880 00000100:00000001:1.0:1787829752.679535:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829752.679537:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829752.679539:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787829752.679544:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829752/real 1787829752] req@ffff8d30d4d270c0 x1874670646861568/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829777 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787829752.679553:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787829752.679554:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787829752.679557:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787829752.679558:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787829752.679560:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829752.679562:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829752.679563:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787829752.679568:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d270c0 x1874670646861568/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829777 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787829752.679576:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829752.679578:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787829752.679580:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829752.679582:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829752.679584:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829752.679586:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787829752.679587:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787829752.679590:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829752.679594:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787829752.679596:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787829752.679630:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d270c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646861568:192.168.203.143@tcp:8:kworker.0 00000100:00000001:1.0:1787829752.679635:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829752.679638:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829752.679639:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787829752.679640:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787829752.679642:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787829752.679643:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787829752.679644:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787829752.679646:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787829752.679646:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787829752.679649:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787829752.679650:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829752.679651:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829752.679653:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787829752.679654:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829752.679656:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787829752.679657:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829753.574694:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829753.574699:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829753.574748:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829753.574752:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829753.574755:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829754.598669:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829754.598674:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829754.598720:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829754.598723:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829754.598726:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829755.622722:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829755.622729:0:815:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1787829755.622731:0:815:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1787829755.622733:0:815:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829755.622737:0:815:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1787829755.622739:0:815:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829755.622741:0:815:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829755.622742:0:815:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829755.622798:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829755.622803:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829755.622807:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829756.646710:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829756.646715:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829756.646755:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829756.646760:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829756.646764:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829757.670710:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829757.670715:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829757.670788:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829757.670793:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829757.670797:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787829757.798733:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829757.798758:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1787829757.798762:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787829757.798766:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787829757.798773:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787829757.798781:0:40798:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 4558 00000100:00080000:0.0:1787829757.798793:0:40798:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787829757.798797:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829757.798799:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787829757.798804:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787829757.798807:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787829757.798811:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:0.0:1787829757.798814:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:0.0:1787829757.798816:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829757.798818:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:0.0:1787829757.798823:0:40798:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787829757.798827:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829757.798846:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787829757.798850:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787829757.798854:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829757.798857:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1787829757.798859:0:40798:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829757.798868:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787829757.798871:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829757.798875:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787829757.798877:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787829757.798882:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829757.798894:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787829757.798905:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d270c0 x1874670646861696/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787829757.798917:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787829757.798923:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829757.798938:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787829757.798942:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829757.798946:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1787829757.798948:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787829757.798951:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787829757.798955:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787829757.798959:0:40798:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4558 00000100:00000001:1.0:1787829757.798962:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00080000:0.0:1787829757.798965:0:40798:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1787829757.798967:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787829757.798970:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787829757.798971:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1787829757.798974:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:1.0:1787829757.798975:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787829757.798977:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787829757.798981:0:40798:0:(genops.c:792:class_conn2export()) Process entered 02000000:00000001:1.0:1787829757.798983:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000020:00000001:0.0:1787829757.798986:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 02000000:00000001:1.0:1787829757.798987:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000020:00000001:0.0:1787829757.798992:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 02000000:00000001:1.0:1787829757.798993:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787829757.799000:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00100000:1.0:1787829757.799000:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d270c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646861696:192.168.203.143@tcp:250:kworker.0 00000100:00000001:0.0:1787829757.799007:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787829757.799008:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:0.0:1787829757.799012:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 02000000:00000001:1.0:1787829757.799013:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787829757.799017:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829757.799019:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00080000:0.0:1787829757.799022:0:40798:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 02000000:00000001:1.0:1787829757.799024:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829757.799029:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829757.799034:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787829757.799040:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829757.799040:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787829757.799043:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601348800 : -126234108202816 : ffff8d30d4d270c0) 02000000:00000001:0.0:1787829757.799046:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829757.799047:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:0.0:1787829757.799052:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1787829757.799057:0:40798:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1787829757.799061:0:40798:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829757.799063:0:40798:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829757.799069:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787829757.799071:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829757.799074:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787829757.799076:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787829757.799078:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829757.799084:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829757.799087:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787829757.799093:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d24000 x1874670646861824/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787829757.799093:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4299232000 00000100:00000001:1.0:1787829757.799104:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1787829757.799106:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787829757.799110:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829757.799111:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4299232000 00000100:00000001:0.0:1787829757.799133:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787829757.799139:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:1.0:1787829757.799140:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 00000100:00000001:1.0:1787829757.799145:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00080000:0.0:1787829757.799147:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:1.0:1787829757.799148:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829757.799153:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:1.0:1787829757.799154:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787829757.799158:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787829757.799159:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829757.799165:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:1.0:1787829757.799167:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00080000:0.0:1787829757.799171:0:40798:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4558 00000100:00000001:1.0:1787829757.799172:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:1.0:1787829757.799175:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787829757.799178:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1787829757.799180:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00080000:0.0:1787829757.799181:0:40798:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1787829757.799182:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787829757.799184:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829757.799188:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 02000000:00000001:1.0:1787829757.799188:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:0.0:1787829757.799194:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 02000000:00000001:1.0:1787829757.799194:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829757.799197:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787829757.799202:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000100:00100000:1.0:1787829757.799205:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d24000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646861824:192.168.203.143@tcp:38:kworker.0 00000020:00000001:0.0:1787829757.799206:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787829757.799212:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:1.0:1787829757.799212:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787829757.799215:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000020:00000001:0.0:1787829757.799219:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 02000000:00000001:1.0:1787829757.799219:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829757.799222:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:0.0:1787829757.799227:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 02000000:00000001:1.0:1787829757.799227:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829757.799234:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787829757.799235:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601336320 : -126234108215296 : ffff8d30d4d24000) 00000100:00000001:1.0:1787829757.799238:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:0.0:1787829757.799243:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:1.0:1787829757.799250:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787829757.799251:0:40798:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:1.0:1787829757.799257:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4299230742 00000100:00000001:0.0:1787829757.799258:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829757.799264:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787829757.799285:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787829757.799288:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829757.799290:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4299232000 00000100:00000001:0.0:1787829757.799291:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1787829757.799297:0:40798:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00000001:1.0:1787829757.799297:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4299230742 00000100:00000001:1.0:1787829757.799303:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4299232000 00000008:00000001:0.0:1787829757.799304:0:40798:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829757.799309:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787829757.799310:0:40798:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829757.799312:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829757.799316:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829757.799318:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:0.0:1787829757.799319:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:1.0:1787829757.799320:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 02000000:00000001:0.0:1787829757.799323:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829757.799324:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:1.0:1787829757.799327:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787829757.799329:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787829757.799347:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:1.0:1787829757.799348:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829757.799351:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829757.799353:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787829757.799357:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829757.799364:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787829757.799371:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d24f00 x1874670646861952/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787829757.799381:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787829757.799384:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829757.799417:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787829757.799420:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829757.799424:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1787829757.799426:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787829757.799427:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787829757.799430:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787829757.799433:0:40798:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4558 00000100:00080000:0.0:1787829757.799438:0:40798:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787829757.799441:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829757.799443:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787829757.799446:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787829757.799448:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787829757.799451:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:1.0:1787829757.799451:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787829757.799455:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000020:00000001:0.0:1787829757.799457:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:1.0:1787829757.799458:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1787829757.799463:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787829757.799464:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787829757.799469:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:1.0:1787829757.799469:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787829757.799477:0:40798:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 02000000:00000001:1.0:1787829757.799477:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787829757.799480:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787829757.799482:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829757.799485:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829757.799487:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787829757.799493:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00100000:1.0:1787829757.799493:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d24f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646861952:192.168.203.143@tcp:38:kworker.0 02000000:00000001:0.0:1787829757.799499:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829757.799503:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:0.0:1787829757.799504:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 02000000:00000001:1.0:1787829757.799508:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000008:00100000:0.0:1787829757.799509:0:40798:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 02000000:00000001:1.0:1787829757.799511:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:0.0:1787829757.799515:0:40798:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829757.799516:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:0.0:1787829757.799520:0:40798:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829757.799525:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1787829757.799528:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829757.799529:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829757.799534:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787829757.799536:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787829757.799538:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829757.799539:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787829757.799542:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601340160 : -126234108211456 : ffff8d30d4d24f00) 00000100:00000001:0.0:1787829757.799546:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829757.799547:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00100000:0.0:1787829757.799557:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d25a40 x1874670646862080/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787829757.799564:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829757.799566:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787829757.799570:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4299230742 00000100:00000001:0.0:1787829757.799571:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829757.799575:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4299232000 00000100:00000001:0.0:1787829757.799578:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829757.799580:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4299230742 00000100:00000001:1.0:1787829757.799591:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4299232000 00000100:00000001:1.0:1787829757.799594:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829757.799597:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829757.799623:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829757.799627:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829757.799631:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787829757.799633:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:1.0:1787829757.799636:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787829757.799637:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1787829757.799639:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787829757.799641:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787829757.799643:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829757.799646:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787829757.799648:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829757.799651:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829757.799656:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d25a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646862080:192.168.203.143@tcp:8:kworker.0 00000100:00000001:1.0:1787829757.799660:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787829757.799662:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787829757.799664:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829757.799666:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787829757.799669:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829757.799676:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787829757.799678:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601343040 : -126234108208576 : ffff8d30d4d25a40) 00000100:00000001:1.0:1787829757.799681:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:1.0:1787829757.799694:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829757.799698:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4299230756 00000100:00000001:1.0:1787829757.799700:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4299232000 00000100:00000001:1.0:1787829757.799703:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4299230756 00000100:00000001:1.0:1787829757.799705:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4299232000 00000100:00000001:1.0:1787829757.799707:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829757.799709:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829757.799712:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829757.799714:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829757.799717:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787829757.799719:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:1.0:1787829757.799721:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787829757.799722:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1787829757.799724:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829757.799726:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829758.694677:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829758.694682:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829758.694728:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829758.694732:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829758.694734:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829759.718662:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829759.718667:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829759.718760:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829759.718763:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829759.718766:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829760.742715:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829760.742722:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829760.742785:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829760.742788:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829760.742790:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:2.0:1787829760.870860:0:2381:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.143:988 00000400:00000100:2.0:1787829760.870893:0:2381:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.143@tcp at host 192.168.203.143 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1787829760.870906:0:2381:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.143@tcp 00000100:00000001:2.0:1787829760.870917:0:2381:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1787829760.870935:0:2381:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787829760.870938:0:2381:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829760.870941:0:2381:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:2.0:1787829760.870951:0:2381:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.143@tcp 00000100:00000001:2.0:1787829760.870956:0:2381:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1787829760.870959:0:2381:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787829760.870960:0:2381:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829760.870962:0:2381:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:2.0:1787829760.870968:0:2381:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.143@tcp 00000100:00000001:2.0:1787829760.870973:0:2381:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1787829760.870976:0:2381:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787829760.870978:0:2381:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829760.870980:0:2381:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787829760.870984:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000800:00000100:2.0:1787829760.870985:0:2381:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.143@tcp 00000100:00000001:1.0:1787829760.870990:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1787829760.870993:0:2381:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787829760.870995:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00000001:2.0:1787829760.870997:0:2381:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787829760.871001:0:2381:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829760.871003:0:2381:0:(events.c:68:request_out_callback()) Process leaving 00000100:00100000:1.0:1787829760.871011:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829757/real 1787829760] req@ffff8d30d4d24f00 x1874670646861952/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829782 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787829760.871028:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787829760.871030:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787829760.871036:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787829760.871039:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787829760.871041:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829760.871044:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829760.871046:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787829760.871051:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d24f00 x1874670646861952/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829782 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787829760.871067:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829760.871070:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787829760.871073:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829760.871074:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829760.871076:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829760.871079:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787829760.871081:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787829760.871085:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829760.871090:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787829760.871093:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787829760.871097:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d24f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646861952:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787829760.871102:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787829760.871107:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829757/real 1787829760] req@ffff8d30d4d24000 x1874670646861824/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829782 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787829760.871117:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787829760.871118:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787829760.871121:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787829760.871123:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787829760.871125:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829760.871133:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829760.871135:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787829760.871140:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d24000 x1874670646861824/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829782 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787829760.871148:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829760.871150:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787829760.871153:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829760.871155:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829760.871156:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829760.871159:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787829760.871161:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787829760.871164:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829760.871168:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787829760.871170:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787829760.871174:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d24000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646861824:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787829760.871178:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787829760.871183:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829757/real 1787829760] req@ffff8d30d4d270c0 x1874670646861696/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829782 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787829760.871192:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787829760.871193:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787829760.871203:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787829760.871205:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787829760.871206:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829760.871208:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829760.871210:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787829760.871214:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d270c0 x1874670646861696/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829782 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787829760.871222:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829760.871224:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787829760.871227:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829760.871229:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829760.871230:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829760.871232:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787829760.871234:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787829760.871237:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829760.871240:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787829760.871242:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787829760.871246:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d270c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646861696:192.168.203.143@tcp:250:kworker.0 00000100:00000001:1.0:1787829760.871253:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829760.871256:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829760.871258:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787829760.871261:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787829760.871266:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787829760.871270:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787829760.871272:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787829760.871276:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787829760.871279:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787829760.871285:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787829760.871287:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829760.871289:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829760.871290:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787829760.871292:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787829760.871294:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787829760.871296:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787829760.871297:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787829760.871298:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787829760.871300:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787829760.871301:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787829760.871303:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829760.871304:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829760.871305:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787829760.871307:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787829760.871310:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787829760.871312:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787829760.871313:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787829760.871314:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787829760.871315:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787829760.871318:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787829760.871320:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829760.871322:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829760.871324:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787829760.871325:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1787829760.871327:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787829760.871332:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829757/real 1787829760] req@ffff8d30d4d25a40 x1874670646862080/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829782 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787829760.871341:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787829760.871343:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787829760.871346:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787829760.871348:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787829760.871350:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829760.871352:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829760.871353:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787829760.871357:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d25a40 x1874670646862080/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829782 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787829760.871365:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829760.871367:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787829760.871369:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829760.871371:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829760.871372:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829760.871374:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787829760.871377:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787829760.871379:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829760.871383:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787829760.871385:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787829760.871396:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d25a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646862080:192.168.203.143@tcp:8:kworker.0 00000100:00000001:1.0:1787829760.871400:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829760.871401:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829760.871402:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787829760.871404:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787829760.871406:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787829760.871408:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787829760.871409:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787829760.871411:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787829760.871412:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787829760.871415:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787829760.871416:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829760.871418:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829760.871426:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787829760.871428:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829760.871430:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787829760.871431:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829761.766665:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829761.766671:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829761.766720:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829761.766723:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829761.766726:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829762.790721:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829762.790728:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829762.790790:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829762.790806:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829762.790810:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787829762.918674:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829762.918682:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1787829762.918684:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787829762.918687:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787829762.918690:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787829762.918696:0:40798:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 4563 00000100:00080000:0.0:1787829762.918704:0:40798:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787829762.918707:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829762.918709:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787829762.918713:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787829762.918715:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787829762.918717:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:0.0:1787829762.918719:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:0.0:1787829762.918720:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829762.918721:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:0.0:1787829762.918724:0:40798:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787829762.918726:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829762.918728:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787829762.918730:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787829762.918732:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829762.918734:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1787829762.918736:0:40798:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829762.918743:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787829762.918744:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829762.918747:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787829762.918749:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787829762.918752:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829762.918761:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787829762.918767:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d252c0 x1874670646862208/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787829762.918775:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787829762.918779:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829762.918790:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787829762.918811:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829762.918813:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1787829762.918815:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787829762.918816:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787829762.918818:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787829762.918821:0:40798:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4563 00000100:00080000:0.0:1787829762.918825:0:40798:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787829762.918827:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829762.918828:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787829762.918829:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787829762.918830:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787829762.918833:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:1.0:1787829762.918833:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:0.0:1787829762.918838:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:0.0:1787829762.918840:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787829762.918840:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787829762.918845:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:1.0:1787829762.918846:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00080000:0.0:1787829762.918848:0:40798:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:1.0:1787829762.918850:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787829762.918851:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829762.918854:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1787829762.918855:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829762.918858:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787829762.918862:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829762.918862:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:0.0:1787829762.918867:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 02000000:00000001:1.0:1787829762.918868:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000008:00100000:0.0:1787829762.918872:0:40798:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 02000000:00000001:1.0:1787829762.918873:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:0.0:1787829762.918880:0:40798:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829762.918884:0:40798:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829762.918884:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d252c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646862208:192.168.203.143@tcp:250:kworker.0 02000000:00000001:0.0:1787829762.918892:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:1.0:1787829762.918893:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787829762.918895:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829762.918899:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:0.0:1787829762.918900:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1787829762.918904:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829762.918905:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787829762.918908:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829762.918910:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:0.0:1787829762.918916:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829762.918918:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787829762.918924:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d25680 x1874670646862336/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787829762.918934:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787829762.918936:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787829762.918937:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829762.918940:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601341120 : -126234108210496 : ffff8d30d4d252c0) 00000100:00000001:0.0:1787829762.918943:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829762.918945:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00080000:0.0:1787829762.918948:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829762.918953:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1787829762.918955:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787829762.918956:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787829762.918965:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787829762.918969:0:40798:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4563 00000100:00080000:0.0:1787829762.918976:0:40798:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787829762.918978:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829762.918980:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787829762.918982:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787829762.918984:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787829762.918988:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000800:00000100:1.0:1787829762.918989:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000020:00000001:0.0:1787829762.918993:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:0.0:1787829762.918998:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829762.919000:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:1.0:1787829762.919003:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00080000:0.0:1787829762.919006:0:40798:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:1.0:1787829762.919009:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787829762.919011:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829762.919013:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829762.919016:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1787829762.919017:0:2390:0:(events.c:68:request_out_callback()) Process leaving 02000000:00000001:0.0:1787829762.919022:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829762.919025:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829762.919027:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829762.919031:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:1.0:1787829762.919032:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4299237120 00000008:00100000:0.0:1787829762.919034:0:40798:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1787829762.919039:0:40798:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829762.919041:0:40798:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829762.919046:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787829762.919047:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829762.919049:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787829762.919051:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:1.0:1787829762.919052:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 02000000:00000001:0.0:1787829762.919054:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829762.919060:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4299237120 00000100:00000001:0.0:1787829762.919061:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829762.919068:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 00000100:00100000:0.0:1787829762.919071:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d27c00 x1874670646862464/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787829762.919074:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829762.919080:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829762.919085:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00000001:0.0:1787829762.919088:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787829762.919093:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829762.919095:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829762/real 1787829762] req@ffff8d30d4d252c0 x1874670646862208/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829787 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787829762.919097:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787829762.919104:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829762.919114:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1787829762.919117:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787829762.919121:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787829762.919128:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000400:00000001:1.0:1787829762.919133:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00080000:0.0:1787829762.919135:0:40798:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4563 00000400:00000001:1.0:1787829762.919137:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00080000:0.0:1787829762.919143:0:40798:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1787829762.919144:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787829762.919149:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787829762.919150:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787829762.919153:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:1.0:1787829762.919155:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1787829762.919159:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:1.0:1787829762.919161:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000020:00000001:0.0:1787829762.919163:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:1.0:1787829762.919166:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1787829762.919167:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00100000:1.0:1787829762.919174:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d252c0 x1874670646862208/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829787 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:0.0:1787829762.919175:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:0.0:1787829762.919180:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829762.919183:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:0.0:1787829762.919192:0:40798:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:1.0:1787829762.919192:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829762.919196:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829762.919198:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787829762.919202:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787829762.919203:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829762.919206:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1787829762.919210:0:40798:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00000001:1.0:1787829762.919212:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000008:00000001:0.0:1787829762.919216:0:40798:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829762.919218:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829762.919220:0:40798:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829762.919222:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829762.919225:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:0.0:1787829762.919227:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:1.0:1787829762.919229:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 02000000:00000001:0.0:1787829762.919231:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829762.919234:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787829762.919235:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787829762.919239:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00080000:1.0:1787829762.919241:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 02000000:00000001:0.0:1787829762.919243:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1787829762.919249:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787829762.919251:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829762.919254:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787829762.919260:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d26940 x1874670646862592/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1787829762.919264:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d252c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646862208:192.168.203.143@tcp:250:kworker.0 00000100:00000001:1.0:1787829762.919274:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829762.919278:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787829762.919279:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829762.919282:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 00000100:00000001:0.0:1787829762.919284:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829762.919286:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 00000100:00000001:0.0:1787829762.919289:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829762.919292:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787829762.919296:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787829762.919299:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787829762.919304:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787829762.919306:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787829762.919310:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787829762.919312:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829762.919315:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829762.919323:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787829762.919325:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829762.919327:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787829762.919329:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1787829762.919330:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787829762.919331:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787829762.919333:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829762.919335:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787829762.919337:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829762.919339:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829762.919343:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d25680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646862336:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787829762.919347:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787829762.919349:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787829762.919351:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829762.919353:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787829762.919355:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829762.919362:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787829762.919363:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601342080 : -126234108209536 : ffff8d30d4d25680) 00000100:00000001:1.0:1787829762.919368:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1787829762.919382:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787829762.919386:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787829762.919388:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829762.919389:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829762.919391:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787829762.919394:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829762.919399:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4299235862 00000100:00000001:1.0:1787829762.919402:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4299237120 00000100:00000001:1.0:1787829762.919405:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4299235862 00000100:00000001:1.0:1787829762.919407:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4299237120 00000100:00000001:1.0:1787829762.919409:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829762.919412:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829762.919413:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787829762.919417:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829762/real 1787829762] req@ffff8d30d4d25680 x1874670646862336/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829787 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787829762.919434:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787829762.919436:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787829762.919439:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787829762.919440:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787829762.919442:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829762.919444:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829762.919445:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787829762.919449:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d25680 x1874670646862336/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829787 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787829762.919457:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829762.919459:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787829762.919462:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829762.919464:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829762.919466:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829762.919467:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787829762.919469:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787829762.919472:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829762.919476:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787829762.919478:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787829762.919482:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d25680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646862336:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787829762.919487:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787829762.919488:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787829762.919490:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829762.919492:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787829762.919494:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829762.919496:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829762.919499:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d27c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646862464:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787829762.919502:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787829762.919504:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787829762.919506:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829762.919508:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787829762.919511:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829762.919517:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787829762.919519:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601351680 : -126234108199936 : ffff8d30d4d27c00) 00000100:00000001:1.0:1787829762.919523:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1787829762.919536:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787829762.919541:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787829762.919544:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829762.919545:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829762.919547:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787829762.919550:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829762.919554:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4299235862 00000100:00000001:1.0:1787829762.919557:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4299237120 00000100:00000001:1.0:1787829762.919560:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4299235862 00000100:00000001:1.0:1787829762.919562:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4299237120 00000100:00000001:1.0:1787829762.919565:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829762.919567:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829762.919569:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787829762.919574:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829762/real 1787829762] req@ffff8d30d4d27c00 x1874670646862464/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829787 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787829762.919584:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787829762.919586:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787829762.919589:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787829762.919591:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787829762.919593:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829762.919596:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829762.919597:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787829762.919629:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d27c00 x1874670646862464/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829787 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787829762.919638:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829762.919640:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787829762.919644:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829762.919646:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829762.919647:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829762.919650:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787829762.919652:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787829762.919655:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829762.919659:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787829762.919661:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787829762.919665:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d27c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646862464:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787829762.919669:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787829762.919671:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787829762.919672:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829762.919675:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787829762.919676:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829762.919679:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829762.919682:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d26940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646862592:192.168.203.143@tcp:8:kworker.0 00000100:00000001:1.0:1787829762.919685:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787829762.919687:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787829762.919688:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829762.919690:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787829762.919693:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829762.919699:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787829762.919700:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601346880 : -126234108204736 : ffff8d30d4d26940) 00000100:00000001:1.0:1787829762.919704:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1787829762.919717:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787829762.919721:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787829762.919723:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829762.919725:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829762.919726:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787829762.919729:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829762.919732:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4299235876 00000100:00000001:1.0:1787829762.919734:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4299237120 00000100:00000001:1.0:1787829762.919737:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4299235876 00000100:00000001:1.0:1787829762.919739:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4299237120 00000100:00000001:1.0:1787829762.919742:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829762.919743:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829762.919745:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787829762.919749:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829762/real 1787829762] req@ffff8d30d4d26940 x1874670646862592/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829787 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787829762.919756:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787829762.919757:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787829762.919760:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787829762.919761:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787829762.919763:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829762.919765:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829762.919766:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787829762.919770:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d26940 x1874670646862592/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829787 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787829762.919776:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829762.919779:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787829762.919781:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829762.919782:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829762.919783:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829762.919785:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787829762.919787:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787829762.919789:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829762.919806:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787829762.919808:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787829762.919811:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d26940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646862592:192.168.203.143@tcp:8:kworker.0 00000100:00000001:1.0:1787829762.919815:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829762.919817:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829762.919819:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787829762.919820:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787829762.919823:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787829762.919825:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787829762.919826:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787829762.919829:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787829762.919831:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787829762.919833:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787829762.919834:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829762.919836:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829762.919837:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787829762.919839:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787829762.919842:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787829762.919843:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787829762.919844:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787829762.919846:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787829762.919847:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787829762.919849:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787829762.919851:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829762.919852:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829762.919854:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787829762.919855:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787829762.919856:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787829762.919857:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787829762.919858:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787829762.919860:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787829762.919861:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787829762.919863:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787829762.919864:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829762.919866:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829762.919870:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787829762.919872:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829762.919874:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787829762.919876:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829763.814672:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829763.814676:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829763.814714:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829763.814717:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829763.814720:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829764.838702:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829764.838707:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829764.838755:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829764.838759:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829764.838761:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829765.862711:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829765.862719:0:815:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1787829765.862722:0:815:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1787829765.862725:0:815:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829765.862730:0:815:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1787829765.862732:0:815:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829765.862734:0:815:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829765.862737:0:815:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829765.862816:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829765.862821:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829765.862824:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829766.886746:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829766.886754:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829766.886856:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829766.886860:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829766.886864:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829767.910747:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829767.910754:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829767.910844:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829767.910848:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829767.910852:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787829768.038718:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829768.038726:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1787829768.038728:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1787829768.038731:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787829768.038735:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1787829768.038749:0:468:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 4568 00000100:00080000:1.0:1787829768.038771:0:468:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1787829768.038775:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787829768.038777:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:1.0:1787829768.038781:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:1.0:1787829768.038783:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:1.0:1787829768.038787:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:1.0:1787829768.038790:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:1.0:1787829768.038792:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787829768.038794:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:1.0:1787829768.038798:0:468:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:1.0:1787829768.038800:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829768.038802:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1787829768.038805:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829768.038807:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829768.038809:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:1.0:1787829768.038811:0:468:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829768.038821:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1787829768.038822:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829768.038825:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1787829768.038827:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1787829768.038830:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829768.038839:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829768.038846:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d13ef480 x1874670646862720/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787829768.038854:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787829768.038859:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829768.038872:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1787829768.038874:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829768.038882:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1787829768.038883:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1787829768.038884:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787829768.038887:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1787829768.038888:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4568 00000100:00080000:1.0:1787829768.038894:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1787829768.038896:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787829768.038897:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:1.0:1787829768.038900:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:1.0:1787829768.038901:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:1.0:1787829768.038903:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000020:00000001:1.0:1787829768.038904:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:1.0:1787829768.038906:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787829768.038907:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:3.0:1787829768.038908:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00080000:1.0:1787829768.038910:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:3.0:1787829768.038913:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1787829768.038914:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829768.038918:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:3.0:1787829768.038918:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787829768.038921:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 02000000:00000001:1.0:1787829768.038923:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829768.038924:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829768.038930:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829768.038931:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:1.0:1787829768.038932:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 02000000:00000001:3.0:1787829768.038935:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000008:00100000:1.0:1787829768.038939:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 02000000:00000001:3.0:1787829768.038939:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:1.0:1787829768.038945:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829768.038947:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d13ef480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646862720:192.168.203.143@tcp:250:kworker.0 00000100:00000001:1.0:1787829768.038950:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829768.038953:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1787829768.038957:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787829768.038958:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787829768.038960:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829768.038962:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829768.038963:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:1.0:1787829768.038967:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787829768.038969:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829768.038971:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1787829768.038975:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829768.038982:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829768.038984:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787829768.038987:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839541351552 : -126234168200064 : ffff8d30d13ef480) 00000100:00000001:3.0:1787829768.038990:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00100000:1.0:1787829768.038992:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d13ef840 x1874670646862848/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787829768.039002:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787829768.039004:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829768.039008:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1787829768.039011:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829768.039015:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:1.0:1787829768.039017:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1787829768.039019:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787829768.039021:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1787829768.039024:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4568 00000100:00080000:1.0:1787829768.039029:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000800:00000100:3.0:1787829768.039032:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787829768.039034:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787829768.039038:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:1.0:1787829768.039041:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:1.0:1787829768.039043:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:3.0:1787829768.039045:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000020:00000001:1.0:1787829768.039046:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:3.0:1787829768.039050:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829768.039052:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829768.039053:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1787829768.039057:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829768.039061:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:3.0:1787829768.039063:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4299242240 00000100:00000001:1.0:1787829768.039067:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787829768.039070:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:3.0:1787829768.039073:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 00000100:00080000:1.0:1787829768.039078:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:3.0:1787829768.039081:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4299242240 00000100:00000001:1.0:1787829768.039082:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829768.039085:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 02000000:00000001:1.0:1787829768.039086:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:3.0:1787829768.039089:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829768.039092:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829768.039093:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829768.039097:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829768.039099:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:3.0:1787829768.039100:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000008:00100000:1.0:1787829768.039102:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:1.0:1787829768.039107:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829768.039108:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829768/real 1787829768] req@ffff8d30d13ef480 x1874670646862720/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829793 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787829768.039111:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829768.039120:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000400:00000001:3.0:1787829768.039122:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 02000000:00000001:1.0:1787829768.039124:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:3.0:1787829768.039124:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829768.039128:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787829768.039129:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:3.0:1787829768.039132:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 02000000:00000001:1.0:1787829768.039133:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:3.0:1787829768.039134:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:1.0:1787829768.039138:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829768.039138:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829768.039139:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829768.039146:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d13ef480 x1874670646862720/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829793 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787829768.039148:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829768.039158:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d13eed00 x1874670646862976/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787829768.039164:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829768.039172:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787829768.039173:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787829768.039178:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829768.039179:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829768.039182:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829768.039183:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00080000:1.0:1787829768.039187:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1787829768.039187:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829768.039193:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00080000:1.0:1787829768.039195:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1787829768.039197:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1787829768.039199:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787829768.039201:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:3.0:1787829768.039203:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787829768.039206:0:468:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4568 00000100:00080000:3.0:1787829768.039210:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829768.039213:0:468:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1787829768.039217:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787829768.039218:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829768.039220:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000001:1.0:1787829768.039222:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00100000:3.0:1787829768.039226:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d13ef480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646862720:192.168.203.143@tcp:250:kworker.0 00000020:00000001:1.0:1787829768.039228:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:1.0:1787829768.039231:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:3.0:1787829768.039232:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:1.0:1787829768.039234:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:3.0:1787829768.039235:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829768.039237:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 00000020:00000001:1.0:1787829768.039240:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 02000000:00000001:3.0:1787829768.039241:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 00000100:00000001:1.0:1787829768.039246:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 02000000:00000001:3.0:1787829768.039246:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00000001:1.0:1787829768.039250:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:3.0:1787829768.039251:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829768.039253:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 00000100:00080000:1.0:1787829768.039259:0:468:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 02000000:00000001:3.0:1787829768.039259:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829768.039262:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787829768.039265:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829768.039268:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1787829768.039271:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829768.039274:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829768.039275:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787829768.039276:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:3.0:1787829768.039278:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000008:00100000:1.0:1787829768.039281:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00000001:3.0:1787829768.039283:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000008:00000001:1.0:1787829768.039286:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829768.039290:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829768.039294:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 02000000:00000001:1.0:1787829768.039295:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:3.0:1787829768.039298:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829768.039299:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829768.039303:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787829768.039304:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1787829768.039308:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:3.0:1787829768.039309:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 02000000:00000001:1.0:1787829768.039311:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829768.039313:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787829768.039316:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787829768.039317:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829768.039319:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829768.039320:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787829768.039323:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829768.039325:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829768.039329:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d13ede00 x1874670646863104/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787829768.039331:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d13ef840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646862848:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829768.039337:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1787829768.039340:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1787829768.039342:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829768.039346:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:1.0:1787829768.039350:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 02000000:00000001:3.0:1787829768.039351:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829768.039356:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829768.039358:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787829768.039361:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839541352512 : -126234168199104 : ffff8d30d13ef840) 00000100:00000001:1.0:1787829768.039362:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829768.039365:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1787829768.039379:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:3.0:1787829768.039383:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787829768.039392:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829768.039393:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829768.039395:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1787829768.039397:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829768.039401:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4299240982 00000100:00000001:3.0:1787829768.039403:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4299242240 00000100:00000001:3.0:1787829768.039405:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4299240982 00000100:00000001:3.0:1787829768.039407:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4299242240 00000100:00000001:3.0:1787829768.039410:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829768.039412:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829768.039414:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1787829768.039418:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829768/real 1787829768] req@ffff8d30d13ef840 x1874670646862848/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829793 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1787829768.039428:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787829768.039429:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829768.039432:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829768.039434:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829768.039435:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829768.039437:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829768.039439:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829768.039443:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d13ef840 x1874670646862848/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829793 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829768.039451:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829768.039454:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829768.039457:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829768.039459:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829768.039460:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829768.039462:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829768.039463:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829768.039465:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829768.039468:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829768.039469:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829768.039471:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d13ef840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646862848:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829768.039475:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787829768.039476:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787829768.039478:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829768.039489:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787829768.039491:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829768.039493:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829768.039497:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d13eed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646862976:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829768.039500:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1787829768.039502:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1787829768.039504:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829768.039506:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1787829768.039509:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829768.039515:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787829768.039516:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839541349632 : -126234168201984 : ffff8d30d13eed00) 00000100:00000001:3.0:1787829768.039520:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1787829768.039533:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:3.0:1787829768.039536:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787829768.039538:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829768.039539:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829768.039540:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1787829768.039542:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829768.039545:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4299240982 00000100:00000001:3.0:1787829768.039553:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4299242240 00000100:00000001:3.0:1787829768.039556:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4299240982 00000100:00000001:3.0:1787829768.039558:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4299242240 00000100:00000001:3.0:1787829768.039561:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829768.039563:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829768.039565:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1787829768.039569:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829768/real 1787829768] req@ffff8d30d13eed00 x1874670646862976/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829793 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1787829768.039579:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787829768.039580:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829768.039583:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829768.039585:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829768.039587:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829768.039589:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829768.039591:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829768.039595:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d13eed00 x1874670646862976/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829793 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829768.039626:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829768.039628:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829768.039630:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829768.039631:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829768.039632:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829768.039634:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829768.039635:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829768.039637:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829768.039640:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829768.039641:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829768.039644:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d13eed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646862976:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829768.039647:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829768.039648:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829768.039650:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829768.039651:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829768.039653:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829768.039654:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829768.039655:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829768.039657:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829768.039658:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829768.039659:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829768.039660:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829768.039661:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829768.039662:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829768.039663:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829768.039664:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829768.039664:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829768.039665:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829768.039666:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829768.039667:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829768.039669:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829768.039669:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829768.039670:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829768.039673:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787829768.039674:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829768.039675:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1787829768.039676:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787829768.039677:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787829768.039678:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787829768.039678:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829768.039680:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787829768.039681:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829768.039683:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829768.039684:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d13ede00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646863104:192.168.203.143@tcp:8:kworker.0 00000100:00000001:3.0:1787829768.039687:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1787829768.039688:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1787829768.039689:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829768.039691:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1787829768.039693:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829768.039697:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787829768.039698:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839541345792 : -126234168205824 : ffff8d30d13ede00) 00000100:00000001:3.0:1787829768.039701:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1787829768.039711:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:3.0:1787829768.039714:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787829768.039715:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829768.039716:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829768.039717:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1787829768.039719:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829768.039722:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4299240996 00000100:00000001:3.0:1787829768.039723:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4299242240 00000100:00000001:3.0:1787829768.039725:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4299240996 00000100:00000001:3.0:1787829768.039726:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4299242240 00000100:00000001:3.0:1787829768.039728:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829768.039729:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829768.039730:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1787829768.039733:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829768/real 1787829768] req@ffff8d30d13ede00 x1874670646863104/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829793 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1787829768.039738:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787829768.039738:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829768.039740:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829768.039741:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829768.039742:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829768.039743:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829768.039744:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829768.039746:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d13ede00 x1874670646863104/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829793 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829768.039751:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829768.039752:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829768.039753:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829768.039754:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829768.039755:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829768.039771:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829768.039773:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829768.039774:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829768.039777:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829768.039779:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829768.039781:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d13ede00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646863104:192.168.203.143@tcp:8:kworker.0 00000100:00000001:3.0:1787829768.039783:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829768.039785:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829768.039786:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829768.039787:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829768.039788:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829768.039789:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829768.039790:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829768.039791:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829768.039792:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829768.039794:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829768.039795:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829768.039796:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829768.039798:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787829768.039799:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829768.039801:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1787829768.039802:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829768.934775:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829768.934785:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829768.934877:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829768.934884:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829768.934888:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829769.958697:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829769.958702:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829769.958779:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829769.958783:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829769.958787:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829770.982690:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829770.982695:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829770.982821:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829770.982824:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829770.982827:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829772.006708:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829772.006714:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829772.006775:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829772.006783:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829772.006786:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829773.030719:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829773.030740:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829773.030792:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829773.030796:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829773.030799:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787829773.158689:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829773.158702:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1787829773.158704:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787829773.158707:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787829773.158711:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787829773.158716:0:468:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 4573 00000100:00080000:3.0:1787829773.158737:0:468:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787829773.158740:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829773.158741:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:3.0:1787829773.158744:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1787829773.158747:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1787829773.158749:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:3.0:1787829773.158751:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:3.0:1787829773.158753:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829773.158754:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:3.0:1787829773.158758:0:468:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:3.0:1787829773.158759:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829773.158761:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787829773.158764:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829773.158766:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829773.158767:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:3.0:1787829773.158769:0:468:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829773.158779:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787829773.158780:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829773.158783:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787829773.158785:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787829773.158789:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829773.158798:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829773.158805:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c7016940 x1874670646863232/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787829773.158813:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787829773.158816:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829773.158828:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787829773.158830:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829773.158832:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1787829773.158834:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787829773.158835:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787829773.158837:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787829773.158839:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4573 00000100:00080000:3.0:1787829773.158842:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787829773.158844:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829773.158845:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:3.0:1787829773.158848:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:1.0:1787829773.158850:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:3.0:1787829773.158850:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1787829773.158852:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:1.0:1787829773.158855:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000020:00000001:3.0:1787829773.158856:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:3.0:1787829773.158859:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829773.158860:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:3.0:1787829773.158862:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:1.0:1787829773.158864:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787829773.158864:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829773.158865:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1787829773.158868:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787829773.158870:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829773.158872:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829773.158874:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787829773.158876:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829773.158878:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829773.158879:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 02000000:00000001:1.0:1787829773.158883:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000008:00100000:3.0:1787829773.158885:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1787829773.158889:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c7016940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646863232:192.168.203.143@tcp:250:kworker.0 00000008:00000001:3.0:1787829773.158890:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829773.158895:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829773.158897:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1787829773.158899:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787829773.158901:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829773.158903:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:3.0:1787829773.158903:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1787829773.158907:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829773.158907:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787829773.158911:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829773.158912:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:3.0:1787829773.158917:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829773.158918:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829773.158925:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c7017c00 x1874670646863360/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787829773.158932:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787829773.158933:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787829773.158934:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829773.158936:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369546048 : -126234340005568 : ffff8d30c7016940) 00000100:00000001:3.0:1787829773.158938:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829773.158940:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00080000:3.0:1787829773.158942:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829773.158946:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1787829773.158947:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787829773.158948:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787829773.158950:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787829773.158952:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4573 00000100:00080000:3.0:1787829773.158955:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787829773.158957:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829773.158958:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:3.0:1787829773.158960:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1787829773.158961:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1787829773.158963:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000020:00000001:3.0:1787829773.158964:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:3.0:1787829773.158966:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829773.158966:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:3.0:1787829773.158968:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:3.0:1787829773.158970:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:1.0:1787829773.158974:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 02000000:00000001:3.0:1787829773.158974:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787829773.158977:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829773.158980:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829773.158982:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:3.0:1787829773.158985:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00000001:1.0:1787829773.158986:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000008:00000001:3.0:1787829773.158987:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829773.158990:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829773.158991:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829773.158994:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829773.158995:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:1.0:1787829773.158996:0:2390:0:(events.c:68:request_out_callback()) Process leaving 02000000:00000001:3.0:1787829773.158997:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829773.159000:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787829773.159001:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:1.0:1787829773.159003:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829773.159003:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829773.159009:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4299247360 00000100:00000001:3.0:1787829773.159011:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829773.159017:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c7014000 x1874670646863488/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787829773.159019:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 00000100:00000001:3.0:1787829773.159027:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787829773.159029:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829773.159030:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787829773.159032:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829773.159034:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1787829773.159035:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4299247360 00000100:00000001:3.0:1787829773.159036:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787829773.159040:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787829773.159041:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 00000100:00000001:3.0:1787829773.159043:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:1.0:1787829773.159046:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00080000:3.0:1787829773.159047:0:468:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4573 00000100:00000001:1.0:1787829773.159050:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787829773.159053:0:468:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1787829773.159054:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00000001:3.0:1787829773.159059:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829773.159060:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00100000:1.0:1787829773.159062:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829773/real 1787829773] req@ffff8d30c7016940 x1874670646863232/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829798 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:3.0:1787829773.159063:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1787829773.159066:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1787829773.159069:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000020:00000001:3.0:1787829773.159074:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000400:00000001:1.0:1787829773.159078:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000001:3.0:1787829773.159078:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000001:1.0:1787829773.159082:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829773.159082:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:1.0:1787829773.159087:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00080000:3.0:1787829773.159088:0:468:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:1.0:1787829773.159090:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829773.159091:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829773.159092:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:3.0:1787829773.159094:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:1.0:1787829773.159096:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 02000000:00000001:3.0:1787829773.159097:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829773.159098:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:3.0:1787829773.159104:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829773.159107:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c7016940 x1874670646863232/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829798 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829773.159108:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:3.0:1787829773.159112:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1787829773.159119:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829773.159123:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829773.159124:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829773.159129:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 02000000:00000001:3.0:1787829773.159131:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:1.0:1787829773.159136:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:3.0:1787829773.159136:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829773.159142:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829773.159143:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:1.0:1787829773.159146:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:3.0:1787829773.159148:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:1.0:1787829773.159151:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 02000000:00000001:3.0:1787829773.159152:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829773.159156:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829773.159160:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1787829773.159162:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829773.159166:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787829773.159170:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787829773.159177:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c7016940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646863232:192.168.203.143@tcp:250:kworker.0 00000100:00000001:1.0:1787829773.159183:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829773.159183:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c7017840 x1874670646863616/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787829773.159187:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829773.159189:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787829773.159192:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787829773.159197:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787829773.159201:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000100:00000001:3.0:1787829773.159201:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000020:00000001:1.0:1787829773.159205:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 00000100:00000001:3.0:1787829773.159208:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829773.159210:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 00000100:00000001:3.0:1787829773.159213:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829773.159214:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787829773.159220:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787829773.159222:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829773.159226:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829773.159234:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787829773.159237:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829773.159239:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787829773.159241:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1787829773.159243:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787829773.159244:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787829773.159246:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829773.159248:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787829773.159250:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829773.159251:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829773.159254:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c7017c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646863360:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787829773.159266:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787829773.159268:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787829773.159270:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829773.159272:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787829773.159275:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829773.159281:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787829773.159283:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369550848 : -126234340000768 : ffff8d30c7017c00) 00000100:00000001:1.0:1787829773.159287:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1787829773.159302:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787829773.159307:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787829773.159310:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829773.159312:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829773.159314:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787829773.159317:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829773.159320:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4299246102 00000100:00000001:1.0:1787829773.159322:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4299247360 00000100:00000001:1.0:1787829773.159324:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4299246102 00000100:00000001:1.0:1787829773.159326:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4299247360 00000100:00000001:1.0:1787829773.159327:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829773.159329:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829773.159330:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787829773.159333:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829773/real 1787829773] req@ffff8d30c7017c00 x1874670646863360/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829798 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787829773.159339:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787829773.159340:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787829773.159341:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787829773.159343:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787829773.159343:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829773.159345:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829773.159346:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787829773.159348:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c7017c00 x1874670646863360/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829798 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787829773.159353:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829773.159354:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787829773.159356:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829773.159357:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829773.159358:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829773.159360:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787829773.159361:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787829773.159363:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829773.159365:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787829773.159366:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787829773.159368:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c7017c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646863360:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787829773.159371:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787829773.159371:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787829773.159372:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829773.159374:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787829773.159375:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829773.159376:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829773.159378:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c7014000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646863488:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787829773.159380:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787829773.159382:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787829773.159383:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829773.159384:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787829773.159386:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829773.159389:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787829773.159390:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369535488 : -126234340016128 : ffff8d30c7014000) 00000100:00000001:1.0:1787829773.159393:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1787829773.159401:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787829773.159404:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787829773.159405:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829773.159406:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829773.159407:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787829773.159409:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829773.159411:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4299246102 00000100:00000001:1.0:1787829773.159413:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4299247360 00000100:00000001:1.0:1787829773.159414:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4299246102 00000100:00000001:1.0:1787829773.159415:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4299247360 00000100:00000001:1.0:1787829773.159418:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829773.159419:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829773.159420:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787829773.159422:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829773/real 1787829773] req@ffff8d30c7014000 x1874670646863488/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829798 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787829773.159427:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787829773.159428:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787829773.159429:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787829773.159430:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787829773.159431:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829773.159432:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829773.159433:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787829773.159437:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c7014000 x1874670646863488/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829798 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787829773.159444:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829773.159446:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787829773.159448:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829773.159450:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829773.159451:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829773.159453:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787829773.159455:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787829773.159457:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829773.159461:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787829773.159463:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787829773.159467:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c7014000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646863488:192.168.203.143@tcp:38:kworker.0 00000100:00000001:1.0:1787829773.159471:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787829773.159472:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:1.0:1787829773.159474:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829773.159476:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:1.0:1787829773.159478:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829773.159480:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829773.159484:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c7017840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646863616:192.168.203.143@tcp:8:kworker.0 00000100:00000001:1.0:1787829773.159487:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:1.0:1787829773.159489:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:1.0:1787829773.159490:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829773.159491:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787829773.159494:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829773.159498:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:1.0:1787829773.159499:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369549888 : -126234340001728 : ffff8d30c7017840) 00000100:00000001:1.0:1787829773.159502:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:1.0:1787829773.159513:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787829773.159517:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787829773.159519:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829773.159521:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829773.159522:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:1.0:1787829773.159525:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829773.159528:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4299246115 00000100:00000001:1.0:1787829773.159531:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4299247360 00000100:00000001:1.0:1787829773.159533:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4299246115 00000100:00000001:1.0:1787829773.159536:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4299247360 00000100:00000001:1.0:1787829773.159538:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:1.0:1787829773.159540:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829773.159542:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:1.0:1787829773.159547:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829773/real 1787829773] req@ffff8d30c7017840 x1874670646863616/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829798 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:1.0:1787829773.159556:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:1.0:1787829773.159557:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787829773.159559:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:1.0:1787829773.159560:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:1.0:1787829773.159561:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829773.159562:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829773.159564:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:1.0:1787829773.159566:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c7017840 x1874670646863616/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829798 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787829773.159576:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829773.159577:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:1.0:1787829773.159579:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829773.159580:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829773.159581:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829773.159582:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:1.0:1787829773.159583:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:1.0:1787829773.159584:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1787829773.159586:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:1.0:1787829773.159587:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:1.0:1787829773.159589:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c7017840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646863616:192.168.203.143@tcp:8:kworker.0 00000100:00000001:1.0:1787829773.159592:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829773.159593:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829773.159594:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787829773.159595:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787829773.159597:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787829773.159598:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787829773.159616:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787829773.159618:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787829773.159619:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787829773.159620:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787829773.159622:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829773.159623:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829773.159624:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787829773.159625:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787829773.159626:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787829773.159627:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787829773.159627:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787829773.159628:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787829773.159630:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787829773.159631:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787829773.159632:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829773.159633:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829773.159634:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:1.0:1787829773.159635:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:1.0:1787829773.159636:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:1.0:1787829773.159637:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:1.0:1787829773.159638:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:1.0:1787829773.159639:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:1.0:1787829773.159639:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:1.0:1787829773.159640:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:1.0:1787829773.159641:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829773.159642:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829773.159645:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:1.0:1787829773.159646:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829773.159647:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787829773.159648:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829774.054680:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829774.054688:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829774.054755:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829774.054759:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829774.054763:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829775.078704:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829775.078731:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829775.078816:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829775.078819:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829775.078821:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829776.102658:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829776.102665:0:815:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1787829776.102668:0:815:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1787829776.102670:0:815:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829776.102675:0:815:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1787829776.102677:0:815:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829776.102678:0:815:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829776.102681:0:815:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829776.102736:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829776.102740:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829776.102744:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829777.126720:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829777.126726:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829777.126779:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829777.126784:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829777.126787:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829778.150701:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829778.150707:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829778.150743:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829778.150747:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829778.150750:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1787829778.278709:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829778.278717:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:1.0:1787829778.278719:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1787829778.278721:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787829778.278725:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1787829778.278730:0:468:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 4578 00000100:00080000:1.0:1787829778.278739:0:468:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1787829778.278741:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787829778.278742:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:1.0:1787829778.278745:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:1.0:1787829778.278747:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:1.0:1787829778.278749:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:1.0:1787829778.278750:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:1.0:1787829778.278751:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787829778.278753:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:1.0:1787829778.278756:0:468:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:1.0:1787829778.278758:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829778.278760:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1787829778.278762:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829778.278764:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829778.278765:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:1.0:1787829778.278767:0:468:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829778.278777:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1787829778.278778:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829778.278780:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1787829778.278782:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1787829778.278785:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829778.278793:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829778.278800:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d13efc00 x1874670646863744/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:1.0:1787829778.278806:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787829778.278809:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829778.278825:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1787829778.278827:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829778.278829:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:1.0:1787829778.278830:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1787829778.278831:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787829778.278833:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1787829778.278835:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4578 00000100:00080000:1.0:1787829778.278838:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1787829778.278840:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787829778.278841:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:1.0:1787829778.278842:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:3.0:1787829778.278843:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:1.0:1787829778.278844:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:1.0:1787829778.278847:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:3.0:1787829778.278848:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000020:00000001:1.0:1787829778.278849:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:3.0:1787829778.278852:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:1.0:1787829778.278853:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787829778.278856:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:3.0:1787829778.278856:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787829778.278859:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1787829778.278861:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:1.0:1787829778.278864:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829778.278864:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787829778.278867:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829778.278870:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829778.278872:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00100000:3.0:1787829778.278875:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d13efc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646863744:192.168.203.143@tcp:250:kworker.0 02000000:00000001:1.0:1787829778.278878:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829778.278881:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829778.278881:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:1.0:1787829778.278884:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 02000000:00000001:3.0:1787829778.278886:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000008:00100000:1.0:1787829778.278888:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 02000000:00000001:3.0:1787829778.278890:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:1.0:1787829778.278891:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829778.278894:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829778.278894:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:1.0:1787829778.278896:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1787829778.278898:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829778.278899:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829778.278900:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:1.0:1787829778.278903:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1787829778.278905:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829778.278909:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829778.278913:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d13ed2c0 x1874670646863872/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787829778.278914:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787829778.278917:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839541353472 : -126234168198144 : ffff8d30d13efc00) 00000100:00000001:1.0:1787829778.278922:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787829778.278923:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:1.0:1787829778.278924:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829778.278928:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1787829778.278930:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829778.278932:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:1.0:1787829778.278933:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1787829778.278934:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787829778.278940:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1787829778.278942:0:468:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4578 00000100:00080000:1.0:1787829778.278947:0:468:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1787829778.278949:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787829778.278949:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:1.0:1787829778.278951:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:1.0:1787829778.278952:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:1.0:1787829778.278953:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000020:00000001:1.0:1787829778.278955:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000800:00000100:3.0:1787829778.278957:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:1.0:1787829778.278958:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787829778.278961:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:1.0:1787829778.278964:0:468:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:1.0:1787829778.278965:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829778.278966:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1787829778.278969:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829778.278971:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829778.278971:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787829778.278974:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:1.0:1787829778.278976:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00000001:3.0:1787829778.278976:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000008:00000001:1.0:1787829778.278980:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829778.278981:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829778.278981:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829778.278984:0:2390:0:(events.c:68:request_out_callback()) Process leaving 02000000:00000001:1.0:1787829778.278987:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1787829778.278989:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829778.278990:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:3.0:1787829778.278990:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829778.278992:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1787829778.278994:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829778.278995:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4299252480 00000100:00000001:1.0:1787829778.278998:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:1.0:1787829778.279002:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d13ed680 x1874670646864000/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787829778.279004:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 00000100:00000001:1.0:1787829778.279010:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:1.0:1787829778.279011:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829778.279013:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:1.0:1787829778.279014:0:468:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1787829778.279016:0:468:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:1.0:1787829778.279017:0:468:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:1.0:1787829778.279017:0:468:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:1.0:1787829778.279019:0:468:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:1.0:1787829778.279020:0:468:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4578 00000100:00000001:3.0:1787829778.279021:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4299252480 00000100:00080000:1.0:1787829778.279024:0:468:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:1.0:1787829778.279027:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787829778.279028:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:3.0:1787829778.279029:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 00000020:00000001:1.0:1787829778.279030:0:468:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:1.0:1787829778.279033:0:468:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:1.0:1787829778.279035:0:468:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:3.0:1787829778.279035:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000020:00000001:1.0:1787829778.279038:0:468:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:3.0:1787829778.279041:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829778.279042:0:468:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:1.0:1787829778.279044:0:468:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:3.0:1787829778.279046:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00080000:1.0:1787829778.279047:0:468:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:1.0:1787829778.279051:0:468:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829778.279052:0:468:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:1.0:1787829778.279054:0:468:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:1.0:1787829778.279056:0:468:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829778.279057:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829778/real 1787829778] req@ffff8d30d13efc00 x1874670646863744/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829803 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:1.0:1787829778.279058:0:468:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:1.0:1787829778.279060:0:468:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000008:00000001:1.0:1787829778.279065:0:468:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829778.279068:0:468:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:1.0:1787829778.279073:0:468:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:1.0:1787829778.279075:0:468:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829778.279076:0:468:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000400:00000001:3.0:1787829778.279076:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 02000000:00000001:1.0:1787829778.279080:0:468:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:1.0:1787829778.279081:0:468:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:3.0:1787829778.279081:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:1.0:1787829778.279086:0:468:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829778.279087:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00100000:1.0:1787829778.279091:0:468:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d13ef0c0 x1874670646864128/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787829778.279092:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829778.279095:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829778.279099:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:1.0:1787829778.279100:0:468:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787829778.279101:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:1.0:1787829778.279103:0:468:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829778.279106:0:468:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829778.279109:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d13efc00 x1874670646863744/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829803 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829778.279115:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829778.279118:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829778.279120:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829778.279122:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829778.279122:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829778.279124:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829778.279125:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829778.279127:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829778.279131:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829778.279133:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829778.279136:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d13efc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646863744:192.168.203.143@tcp:250:kworker.0 00000100:00000001:3.0:1787829778.279139:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829778.279140:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829778.279141:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829778.279143:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829778.279145:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829778.279147:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829778.279148:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829778.279151:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829778.279153:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829778.279156:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829778.279157:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829778.279159:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829778.279165:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787829778.279166:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829778.279167:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1787829778.279168:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787829778.279169:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787829778.279170:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787829778.279171:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829778.279173:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787829778.279174:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829778.279175:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829778.279177:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d13ed2c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646863872:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829778.279179:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1787829778.279180:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1787829778.279181:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829778.279182:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1787829778.279184:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829778.279188:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787829778.279189:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839541342912 : -126234168208704 : ffff8d30d13ed2c0) 00000100:00000001:3.0:1787829778.279192:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1787829778.279201:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:3.0:1787829778.279204:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787829778.279205:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829778.279206:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829778.279207:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1787829778.279209:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829778.279213:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4299251222 00000100:00000001:3.0:1787829778.279215:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4299252480 00000100:00000001:3.0:1787829778.279218:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4299251222 00000100:00000001:3.0:1787829778.279219:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4299252480 00000100:00000001:3.0:1787829778.279222:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829778.279223:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829778.279225:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1787829778.279228:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829778/real 1787829778] req@ffff8d30d13ed2c0 x1874670646863872/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829803 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1787829778.279235:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787829778.279235:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829778.279237:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829778.279238:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829778.279239:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829778.279240:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829778.279241:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829778.279243:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d13ed2c0 x1874670646863872/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829803 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829778.279248:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829778.279249:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829778.279250:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829778.279251:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829778.279252:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829778.279253:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829778.279254:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829778.279256:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829778.279257:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829778.279259:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829778.279261:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d13ed2c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646863872:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829778.279269:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787829778.279270:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787829778.279271:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829778.279272:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787829778.279274:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829778.279275:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829778.279277:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d13ed680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646864000:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829778.279279:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1787829778.279280:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1787829778.279281:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829778.279282:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1787829778.279284:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829778.279287:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787829778.279288:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839541343872 : -126234168207744 : ffff8d30d13ed680) 00000100:00000001:3.0:1787829778.279290:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1787829778.279297:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:3.0:1787829778.279300:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787829778.279301:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829778.279302:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829778.279303:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1787829778.279304:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829778.279306:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4299251222 00000100:00000001:3.0:1787829778.279308:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4299252480 00000100:00000001:3.0:1787829778.279309:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4299251222 00000100:00000001:3.0:1787829778.279310:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4299252480 00000100:00000001:3.0:1787829778.279312:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829778.279313:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829778.279314:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1787829778.279316:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829778/real 1787829778] req@ffff8d30d13ed680 x1874670646864000/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829803 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1787829778.279320:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787829778.279320:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829778.279322:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829778.279322:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829778.279323:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829778.279324:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829778.279325:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829778.279327:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d13ed680 x1874670646864000/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829803 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829778.279331:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829778.279332:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829778.279334:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829778.279335:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829778.279336:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829778.279337:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829778.279337:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829778.279339:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829778.279340:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829778.279341:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829778.279343:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d13ed680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646864000:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829778.279345:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787829778.279345:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787829778.279346:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829778.279347:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787829778.279348:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829778.279349:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829778.279351:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d13ef0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646864128:192.168.203.143@tcp:8:kworker.0 00000100:00000001:3.0:1787829778.279352:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1787829778.279353:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1787829778.279354:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829778.279355:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1787829778.279356:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829778.279359:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787829778.279360:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839541350592 : -126234168201024 : ffff8d30d13ef0c0) 00000100:00000001:3.0:1787829778.279362:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1787829778.279367:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:3.0:1787829778.279369:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787829778.279370:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829778.279371:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829778.279372:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1787829778.279373:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829778.279375:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4299251235 00000100:00000001:3.0:1787829778.279376:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4299252480 00000100:00000001:3.0:1787829778.279378:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4299251235 00000100:00000001:3.0:1787829778.279379:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4299252480 00000100:00000001:3.0:1787829778.279380:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829778.279381:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829778.279382:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1787829778.279384:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829778/real 1787829778] req@ffff8d30d13ef0c0 x1874670646864128/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829803 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1787829778.279388:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787829778.279389:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829778.279390:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829778.279391:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829778.279391:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829778.279392:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829778.279393:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829778.279395:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d13ef0c0 x1874670646864128/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829803 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829778.279398:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829778.279399:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829778.279401:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829778.279402:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829778.279402:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829778.279403:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829778.279404:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829778.279405:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829778.279407:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829778.279408:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829778.279409:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d13ef0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646864128:192.168.203.143@tcp:8:kworker.0 00000100:00000001:3.0:1787829778.279411:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829778.279412:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829778.279413:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829778.279414:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829778.279415:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829778.279416:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829778.279417:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829778.279419:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829778.279420:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829778.279421:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829778.279422:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829778.279423:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829778.279424:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829778.279425:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829778.279426:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829778.279427:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829778.279427:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829778.279428:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829778.279428:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829778.279429:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829778.279430:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829778.279431:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829778.279431:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829778.279432:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829778.279433:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829778.279433:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829778.279434:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829778.279435:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829778.279435:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829778.279436:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829778.279437:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829778.279438:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829778.279439:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787829778.279440:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829778.279441:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1787829778.279442:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829779.174715:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829779.174724:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829779.174788:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829779.174793:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829779.174796:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829780.198731:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829780.198738:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829780.198789:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829780.198792:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829780.198794:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829781.222673:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829781.222682:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829781.222742:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829781.222747:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829781.222750:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829782.246756:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829782.246764:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829782.246819:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829782.246824:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829782.246828:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829783.270672:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829783.270677:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829783.270717:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829783.270720:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829783.270722:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787829783.398738:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829783.398748:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1787829783.398750:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787829783.398754:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787829783.398758:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787829783.398764:0:40798:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 4583 00000100:00080000:0.0:1787829783.398773:0:40798:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787829783.398776:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829783.398778:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787829783.398782:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787829783.398784:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787829783.398787:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:0.0:1787829783.398789:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:0.0:1787829783.398791:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829783.398792:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:0.0:1787829783.398797:0:40798:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787829783.398800:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829783.398802:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787829783.398805:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787829783.398808:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829783.398811:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1787829783.398813:0:40798:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829783.398824:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787829783.398825:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829783.398828:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787829783.398830:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787829783.398833:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829783.398845:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787829783.398853:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d26940 x1874670646864256/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787829783.398862:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787829783.398866:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829783.398881:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787829783.398884:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829783.398887:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1787829783.398888:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787829783.398890:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787829783.398892:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787829783.398895:0:40798:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4583 00000100:00080000:0.0:1787829783.398898:0:40798:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787829783.398900:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829783.398902:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787829783.398904:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787829783.398905:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787829783.398907:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000020:00000001:0.0:1787829783.398909:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:0.0:1787829783.398911:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829783.398913:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:0.0:1787829783.398917:0:40798:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787829783.398920:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829783.398921:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:3.0:1787829783.398924:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:0.0:1787829783.398927:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829783.398928:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787829783.398931:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 02000000:00000001:0.0:1787829783.398932:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829783.398934:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787829783.398936:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:3.0:1787829783.398936:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000008:00100000:0.0:1787829783.398941:0:40798:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 02000000:00000001:3.0:1787829783.398941:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787829783.398944:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000008:00000001:0.0:1787829783.398946:0:40798:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829783.398947:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829783.398951:0:40798:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829783.398954:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d26940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646864256:192.168.203.143@tcp:250:kworker.0 02000000:00000001:0.0:1787829783.398957:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:3.0:1787829783.398958:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787829783.398961:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829783.398962:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:0.0:1787829783.398964:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787829783.398965:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829783.398968:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787829783.398968:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1787829783.398972:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829783.398974:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829783.398979:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787829783.398986:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d27c00 x1874670646864384/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787829783.398987:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787829783.398989:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601346880 : -126234108204736 : ffff8d30d4d26940) 00000100:00000001:3.0:1787829783.398994:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:0.0:1787829783.398999:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787829783.399002:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829783.399006:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787829783.399008:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829783.399012:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1787829783.399014:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787829783.399015:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787829783.399018:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787829783.399020:0:40798:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4583 00000800:00000100:3.0:1787829783.399024:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00080000:0.0:1787829783.399026:0:40798:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787829783.399029:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829783.399031:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787829783.399033:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787829783.399035:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:3.0:1787829783.399035:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000020:00000001:0.0:1787829783.399037:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:3.0:1787829783.399039:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000020:00000001:0.0:1787829783.399041:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:3.0:1787829783.399042:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829783.399047:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829783.399047:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787829783.399050:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:3.0:1787829783.399052:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787829783.399056:0:40798:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:3.0:1787829783.399057:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4299257600 00000100:00000001:0.0:1787829783.399059:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829783.399063:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:3.0:1787829783.399065:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 02000000:00000001:0.0:1787829783.399066:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787829783.399070:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829783.399072:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:3.0:1787829783.399072:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4299257600 00000008:00100000:0.0:1787829783.399077:0:40798:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00000001:3.0:1787829783.399078:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 00000100:00000001:3.0:1787829783.399081:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000008:00000001:0.0:1787829783.399082:0:40798:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829783.399084:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829783.399085:0:40798:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829783.399087:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 02000000:00000001:0.0:1787829783.399091:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00100000:3.0:1787829783.399092:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829783/real 1787829783] req@ffff8d30d4d26940 x1874670646864256/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829808 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:0.0:1787829783.399094:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829783.399098:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787829783.399101:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787829783.399104:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:3.0:1787829783.399106:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787829783.399108:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787829783.399110:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829783.399111:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829783.399113:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829783.399116:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829783.399119:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00100000:0.0:1787829783.399120:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d25680 x1874670646864512/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787829783.399121:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829783.399127:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d26940 x1874670646864256/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829808 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787829783.399141:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787829783.399142:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829783.399145:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829783.399147:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787829783.399148:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829783.399149:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829783.399152:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00080000:0.0:1787829783.399153:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1787829783.399153:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829783.399157:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00080000:0.0:1787829783.399159:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1787829783.399160:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787829783.399164:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787829783.399164:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829783.399167:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1787829783.399169:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829783.399171:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829783.399174:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d26940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646864256:192.168.203.143@tcp:250:kworker.0 00000100:00000001:3.0:1787829783.399179:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829783.399180:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:3.0:1787829783.399184:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00080000:0.0:1787829783.399187:0:40798:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4583 00000100:00000001:3.0:1787829783.399188:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 00000100:00080000:0.0:1787829783.399192:0:40798:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 02000000:00000001:3.0:1787829783.399192:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 00000100:00000001:0.0:1787829783.399196:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 02000000:00000001:3.0:1787829783.399197:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00000001:0.0:1787829783.399198:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:3.0:1787829783.399200:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787829783.399202:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787829783.399204:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1787829783.399204:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 00000020:00000001:0.0:1787829783.399208:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 02000000:00000001:3.0:1787829783.399210:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 00000020:00000001:0.0:1787829783.399211:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 02000000:00000001:3.0:1787829783.399214:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787829783.399216:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829783.399219:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:3.0:1787829783.399220:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829783.399223:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829783.399225:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:0.0:1787829783.399227:0:40798:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787829783.399231:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829783.399232:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 02000000:00000001:0.0:1787829783.399233:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:3.0:1787829783.399236:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829783.399237:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829783.399240:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:0.0:1787829783.399241:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829783.399243:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787829783.399244:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:3.0:1787829783.399247:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000008:00100000:0.0:1787829783.399248:0:40798:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00000001:3.0:1787829783.399250:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000008:00000001:0.0:1787829783.399252:0:40798:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829783.399253:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829783.399255:0:40798:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829783.399258:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787829783.399261:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829783.399263:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829783.399264:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787829783.399268:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829783.399269:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d27c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646864384:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787829783.399271:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787829783.399273:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:3.0:1787829783.399276:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787829783.399277:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829783.399278:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1787829783.399281:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829783.399283:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:0.0:1787829783.399284:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829783.399285:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787829783.399292:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d252c0 x1874670646864640/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787829783.399292:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787829783.399293:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601351680 : -126234108199936 : ffff8d30d4d27c00) 00000100:00000001:3.0:1787829783.399299:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:0.0:1787829783.399308:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000800:00000100:3.0:1787829783.399312:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:0.0:1787829783.399313:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829783.399317:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1787829783.399319:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829783.399320:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829783.399323:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829783.399324:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1787829783.399327:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829783.399329:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4299256342 00000100:00000001:3.0:1787829783.399332:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4299257600 00000100:00000001:3.0:1787829783.399333:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4299256342 00000100:00000001:3.0:1787829783.399335:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4299257600 00000100:00000001:3.0:1787829783.399336:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829783.399337:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829783.399338:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1787829783.399342:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829783/real 1787829783] req@ffff8d30d4d27c00 x1874670646864384/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829808 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1787829783.399347:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787829783.399348:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829783.399350:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829783.399351:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829783.399352:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829783.399353:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829783.399355:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829783.399357:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d27c00 x1874670646864384/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829808 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829783.399361:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829783.399362:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829783.399364:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829783.399365:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829783.399365:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829783.399366:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829783.399368:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829783.399370:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829783.399372:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829783.399374:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829783.399377:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d27c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646864384:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829783.399381:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787829783.399382:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787829783.399383:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829783.399385:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787829783.399387:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829783.399389:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829783.399391:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d25680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646864512:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829783.399394:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1787829783.399396:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1787829783.399397:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829783.399399:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1787829783.399402:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829783.399406:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787829783.399408:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601342080 : -126234108209536 : ffff8d30d4d25680) 00000100:00000001:3.0:1787829783.399411:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1787829783.399422:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:3.0:1787829783.399425:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787829783.399426:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829783.399427:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829783.399428:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1787829783.399430:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829783.399433:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4299256342 00000100:00000001:3.0:1787829783.399434:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4299257600 00000100:00000001:3.0:1787829783.399436:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4299256342 00000100:00000001:3.0:1787829783.399437:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4299257600 00000100:00000001:3.0:1787829783.399438:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829783.399439:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829783.399440:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1787829783.399443:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829783/real 1787829783] req@ffff8d30d4d25680 x1874670646864512/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829808 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1787829783.399447:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787829783.399448:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829783.399449:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829783.399450:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829783.399451:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829783.399452:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829783.399453:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829783.399455:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d25680 x1874670646864512/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829808 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829783.399458:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829783.399459:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829783.399460:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829783.399461:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829783.399462:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829783.399463:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829783.399464:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829783.399465:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829783.399466:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829783.399468:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829783.399470:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d25680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646864512:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829783.399472:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829783.399473:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829783.399474:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829783.399475:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829783.399476:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829783.399477:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829783.399477:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829783.399479:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829783.399479:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829783.399482:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829783.399482:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829783.399483:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829783.399484:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829783.399485:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829783.399486:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829783.399486:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829783.399487:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829783.399488:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829783.399488:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829783.399489:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829783.399490:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829783.399491:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829783.399493:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787829783.399494:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829783.399495:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1787829783.399496:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787829783.399496:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787829783.399497:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787829783.399498:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829783.399499:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787829783.399500:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829783.399501:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829783.399503:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d252c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646864640:192.168.203.143@tcp:8:kworker.0 00000100:00000001:3.0:1787829783.399505:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1787829783.399506:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1787829783.399507:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829783.399508:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1787829783.399509:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829783.399512:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787829783.399513:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601341120 : -126234108210496 : ffff8d30d4d252c0) 00000100:00000001:3.0:1787829783.399515:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1787829783.399521:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:3.0:1787829783.399523:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787829783.399525:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829783.399525:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829783.399526:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1787829783.399528:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829783.399529:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4299256355 00000100:00000001:3.0:1787829783.399530:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4299257600 00000100:00000001:3.0:1787829783.399532:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4299256355 00000100:00000001:3.0:1787829783.399533:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4299257600 00000100:00000001:3.0:1787829783.399534:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829783.399536:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829783.399537:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1787829783.399539:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829783/real 1787829783] req@ffff8d30d4d252c0 x1874670646864640/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829808 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1787829783.399543:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787829783.399543:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829783.399544:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829783.399545:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829783.399546:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829783.399547:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829783.399548:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829783.399550:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d252c0 x1874670646864640/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829808 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829783.399553:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829783.399554:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829783.399556:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829783.399557:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829783.399558:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829783.399559:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829783.399560:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829783.399561:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829783.399563:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829783.399564:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829783.399565:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d252c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646864640:192.168.203.143@tcp:8:kworker.0 00000100:00000001:3.0:1787829783.399567:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829783.399568:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829783.399569:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829783.399569:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829783.399570:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829783.399571:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829783.399571:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829783.399572:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829783.399573:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829783.399574:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829783.399574:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829783.399575:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829783.399577:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787829783.399577:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829783.399579:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1787829783.399580:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829784.294720:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829784.294725:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829784.294770:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829784.294773:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829784.294778:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829785.318678:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829785.318683:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829785.318720:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829785.318724:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829785.318727:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829786.342762:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829786.342772:0:815:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1787829786.342775:0:815:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1787829786.342779:0:815:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829786.342785:0:815:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1787829786.342788:0:815:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829786.342790:0:815:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829786.342793:0:815:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829786.342851:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829786.342855:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829786.342857:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829787.366756:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829787.366761:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829787.366845:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829787.366849:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829787.366854:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829788.390735:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829788.390743:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829788.390827:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829788.390834:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829788.390839:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787829788.518709:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829788.518717:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1787829788.518720:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787829788.518723:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787829788.518727:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787829788.518732:0:40798:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 4589 00000100:00080000:0.0:1787829788.518739:0:40798:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787829788.518742:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829788.518743:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787829788.518746:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787829788.518748:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787829788.518750:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:0.0:1787829788.518752:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:0.0:1787829788.518753:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829788.518754:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:0.0:1787829788.518758:0:40798:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787829788.518760:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829788.518762:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787829788.518764:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787829788.518766:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829788.518768:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1787829788.518770:0:40798:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829788.518778:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787829788.518780:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829788.518782:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787829788.518784:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787829788.518787:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829788.518796:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787829788.518803:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d25a40 x1874670646864768/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787829788.518811:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787829788.518814:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829788.518829:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787829788.518836:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829788.518839:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1787829788.518840:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787829788.518841:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787829788.518843:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787829788.518845:0:40798:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4589 00000100:00080000:0.0:1787829788.518847:0:40798:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787829788.518849:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829788.518850:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787829788.518852:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787829788.518853:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787829788.518855:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000020:00000001:0.0:1787829788.518857:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:0.0:1787829788.518858:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829788.518860:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:0.0:1787829788.518863:0:40798:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787829788.518865:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829788.518866:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:3.0:1787829788.518869:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:0.0:1787829788.518871:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787829788.518874:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829788.518874:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787829788.518875:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:3.0:1787829788.518878:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000008:00100000:0.0:1787829788.518881:0:40798:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00000001:3.0:1787829788.518881:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787829788.518885:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:0.0:1787829788.518886:0:40798:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829788.518889:0:40798:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829788.518890:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787829788.518892:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787829788.518893:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787829788.518894:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829788.518898:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829788.518899:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787829788.518901:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787829788.518904:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829788.518906:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d25a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646864768:192.168.203.143@tcp:250:kworker.0 00000100:00000001:0.0:1787829788.518908:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829788.518912:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00100000:0.0:1787829788.518916:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d270c0 x1874670646864896/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:3.0:1787829788.518918:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1787829788.518921:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829788.518926:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:0.0:1787829788.518929:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 02000000:00000001:3.0:1787829788.518931:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829788.518932:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829788.518937:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787829788.518940:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829788.518942:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1787829788.518943:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787829788.518945:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787829788.518946:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:3.0:1787829788.518947:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00080000:0.0:1787829788.518948:0:40798:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4589 00000100:00000001:3.0:1787829788.518950:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601343040 : -126234108208576 : ffff8d30d4d25a40) 00000100:00080000:0.0:1787829788.518952:0:40798:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787829788.518955:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:0.0:1787829788.518956:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829788.518958:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787829788.518960:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787829788.518961:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787829788.518963:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000020:00000001:0.0:1787829788.518964:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:0.0:1787829788.518966:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829788.518967:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:0.0:1787829788.518969:0:40798:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787829788.518971:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829788.518972:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787829788.518973:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787829788.518975:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829788.518976:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1787829788.518977:0:40798:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1787829788.518979:0:40798:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829788.518980:0:40798:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829788.518983:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787829788.518984:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829788.518985:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787829788.518986:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787829788.518988:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:3.0:1787829788.518990:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:0.0:1787829788.518992:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787829788.518998:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d24000 x1874670646865024/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787829788.519002:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1787829788.519004:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787829788.519007:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829788.519007:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787829788.519010:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829788.519011:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787829788.519013:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829788.519015:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1787829788.519015:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787829788.519016:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787829788.519018:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787829788.519020:0:40798:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4589 00000100:00080000:0.0:1787829788.519023:0:40798:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787829788.519025:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787829788.519026:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829788.519028:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787829788.519030:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787829788.519031:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:3.0:1787829788.519032:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787829788.519033:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000020:00000001:0.0:1787829788.519038:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:0.0:1787829788.519040:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829788.519040:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4299262720 00000100:00000001:0.0:1787829788.519043:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:0.0:1787829788.519046:0:40798:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787829788.519047:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829788.519048:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:3.0:1787829788.519050:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 02000000:00000001:0.0:1787829788.519051:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787829788.519055:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829788.519057:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1787829788.519059:0:40798:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00000001:3.0:1787829788.519060:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4299262720 00000008:00000001:0.0:1787829788.519061:0:40798:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829788.519066:0:40798:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829788.519069:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 02000000:00000001:0.0:1787829788.519070:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787829788.519072:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829788.519074:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787829788.519075:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787829788.519077:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:3.0:1787829788.519079:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829788.519080:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829788.519083:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00000001:0.0:1787829788.519085:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829788.519089:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829788/real 1787829788] req@ffff8d30d4d25a40 x1874670646864768/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829813 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1787829788.519091:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d24f00 x1874670646865152/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787829788.519118:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000400:00000001:3.0:1787829788.519119:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000001:0.0:1787829788.519120:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:3.0:1787829788.519123:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787829788.519124:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829788.519127:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829788.519131:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829788.519132:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829788.519134:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829788.519136:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829788.519140:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d25a40 x1874670646864768/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829813 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829788.519151:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829788.519153:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829788.519155:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829788.519157:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829788.519158:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829788.519160:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829788.519161:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829788.519163:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829788.519166:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829788.519169:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829788.519171:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d25a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646864768:192.168.203.143@tcp:250:kworker.0 00000100:00000001:3.0:1787829788.519175:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829788.519177:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829788.519178:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829788.519181:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829788.519184:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829788.519185:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829788.519187:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829788.519191:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829788.519193:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829788.519197:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829788.519198:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829788.519201:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829788.519209:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787829788.519212:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829788.519215:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1787829788.519218:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787829788.519220:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787829788.519222:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787829788.519224:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829788.519226:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787829788.519228:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829788.519230:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829788.519235:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d270c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646864896:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829788.519240:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1787829788.519242:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1787829788.519243:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829788.519246:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1787829788.519248:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829788.519254:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787829788.519255:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601348800 : -126234108202816 : ffff8d30d4d270c0) 00000100:00000001:3.0:1787829788.519260:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1787829788.519270:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:3.0:1787829788.519274:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787829788.519276:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829788.519277:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829788.519278:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1787829788.519280:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829788.519283:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4299261462 00000100:00000001:3.0:1787829788.519285:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4299262720 00000100:00000001:3.0:1787829788.519286:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4299261462 00000100:00000001:3.0:1787829788.519288:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4299262720 00000100:00000001:3.0:1787829788.519290:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829788.519291:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829788.519292:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1787829788.519295:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829788/real 1787829788] req@ffff8d30d4d270c0 x1874670646864896/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829813 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1787829788.519300:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787829788.519301:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829788.519303:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829788.519304:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829788.519305:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829788.519306:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829788.519307:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829788.519310:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d270c0 x1874670646864896/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829813 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829788.519319:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829788.519320:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829788.519322:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829788.519323:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829788.519324:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829788.519325:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829788.519326:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829788.519328:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829788.519330:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829788.519331:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829788.519333:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d270c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646864896:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829788.519336:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787829788.519337:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787829788.519338:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829788.519339:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787829788.519341:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829788.519342:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829788.519344:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d24000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646865024:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829788.519346:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1787829788.519347:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1787829788.519348:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829788.519349:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1787829788.519351:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829788.519354:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787829788.519355:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601336320 : -126234108215296 : ffff8d30d4d24000) 00000100:00000001:3.0:1787829788.519356:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1787829788.519363:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:3.0:1787829788.519365:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787829788.519367:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829788.519368:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829788.519369:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1787829788.519371:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829788.519373:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4299261462 00000100:00000001:3.0:1787829788.519374:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4299262720 00000100:00000001:3.0:1787829788.519376:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4299261462 00000100:00000001:3.0:1787829788.519377:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4299262720 00000100:00000001:3.0:1787829788.519378:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829788.519380:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829788.519381:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1787829788.519383:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829788/real 1787829788] req@ffff8d30d4d24000 x1874670646865024/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829813 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1787829788.519388:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787829788.519388:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829788.519390:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829788.519390:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829788.519391:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829788.519392:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829788.519393:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829788.519395:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d24000 x1874670646865024/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829813 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829788.519399:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829788.519401:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829788.519402:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829788.519403:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829788.519404:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829788.519405:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829788.519406:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829788.519408:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829788.519409:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829788.519411:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829788.519413:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d24000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646865024:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829788.519415:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787829788.519416:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787829788.519417:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829788.519418:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787829788.519419:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829788.519421:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829788.519422:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d24f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646865152:192.168.203.143@tcp:8:kworker.0 00000100:00000001:3.0:1787829788.519424:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1787829788.519425:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1787829788.519426:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829788.519427:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1787829788.519428:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829788.519431:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787829788.519432:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601340160 : -126234108211456 : ffff8d30d4d24f00) 00000100:00000001:3.0:1787829788.519434:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1787829788.519440:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:3.0:1787829788.519442:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787829788.519443:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829788.519444:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829788.519445:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1787829788.519447:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829788.519448:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4299261475 00000100:00000001:3.0:1787829788.519450:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4299262720 00000100:00000001:3.0:1787829788.519451:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4299261475 00000100:00000001:3.0:1787829788.519453:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4299262720 00000100:00000001:3.0:1787829788.519454:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829788.519455:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829788.519456:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1787829788.519458:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829788/real 1787829788] req@ffff8d30d4d24f00 x1874670646865152/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829813 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1787829788.519467:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787829788.519468:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829788.519469:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829788.519471:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829788.519471:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829788.519472:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829788.519473:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829788.519475:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d24f00 x1874670646865152/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829813 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829788.519479:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829788.519480:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829788.519482:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829788.519483:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829788.519483:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829788.519484:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829788.519485:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829788.519487:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829788.519488:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829788.519490:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829788.519492:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d24f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646865152:192.168.203.143@tcp:8:kworker.0 00000100:00000001:3.0:1787829788.519494:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829788.519495:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829788.519495:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829788.519496:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829788.519498:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829788.519499:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829788.519500:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829788.519501:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829788.519502:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829788.519504:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829788.519505:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829788.519507:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829788.519508:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829788.519509:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829788.519510:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829788.519511:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829788.519511:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829788.519512:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829788.519513:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829788.519514:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829788.519515:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829788.519516:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829788.519517:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829788.519517:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829788.519518:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829788.519519:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829788.519520:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829788.519520:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829788.519521:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829788.519522:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829788.519523:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829788.519524:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829788.519526:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787829788.519527:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829788.519528:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1787829788.519529:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829789.414705:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829789.414712:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829789.414764:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829789.414767:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829789.414770:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829790.438682:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829790.438688:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829790.438742:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829790.438748:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829790.438752:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829791.462687:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829791.462692:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829791.462751:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829791.462756:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829791.462759:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829792.486714:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829792.486720:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829792.486786:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829792.486790:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829792.486794:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829793.510697:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829793.510705:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829793.510767:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829793.510771:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829793.510774:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787829793.638709:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829793.638717:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1787829793.638720:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787829793.638724:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787829793.638727:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787829793.638733:0:40798:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 4594 00000100:00080000:2.0:1787829793.638743:0:40798:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787829793.638745:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829793.638747:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787829793.638751:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787829793.638753:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787829793.638756:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:2.0:1787829793.638759:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:2.0:1787829793.638761:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829793.638762:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:2.0:1787829793.638766:0:40798:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787829793.638768:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829793.638770:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787829793.638774:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787829793.638777:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829793.638779:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:2.0:1787829793.638781:0:40798:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829793.638788:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787829793.638790:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829793.638792:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787829793.638794:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787829793.638798:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829793.638807:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787829793.638815:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2f480 x1874670646865280/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787829793.638823:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787829793.638827:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829793.638842:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787829793.638845:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829793.638848:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1787829793.638849:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787829793.638850:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787829793.638852:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787829793.638855:0:40798:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4594 00000100:00080000:2.0:1787829793.638859:0:40798:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787829793.638860:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829793.638862:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787829793.638864:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787829793.638865:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:0.0:1787829793.638866:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:2.0:1787829793.638868:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:0.0:1787829793.638871:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000020:00000001:2.0:1787829793.638872:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:0.0:1787829793.638874:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1787829793.638875:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829793.638878:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787829793.638879:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:0.0:1787829793.638882:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787829793.638884:0:40798:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 02000000:00000001:0.0:1787829793.638888:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:2.0:1787829793.638888:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829793.638892:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787829793.638892:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787829793.638896:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829793.638899:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00100000:0.0:1787829793.638901:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646865280:192.168.203.143@tcp:250:kworker.0 02000000:00000001:2.0:1787829793.638903:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829793.638908:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1787829793.638910:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787829793.638914:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000008:00100000:2.0:1787829793.638914:0:40798:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 02000000:00000001:0.0:1787829793.638918:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:2.0:1787829793.638919:0:40798:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829793.638921:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:2.0:1787829793.638922:0:40798:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829793.638926:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829793.638927:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787829793.638930:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829793.638931:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787829793.638932:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787829793.638934:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829793.638939:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829793.638941:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00100000:2.0:1787829793.638944:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2da40 x1874670646865408/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787829793.638945:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417881728 : -126234291669888 : ffff8d30c9e2f480) 00000100:00000001:0.0:1787829793.638951:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:2.0:1787829793.638956:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787829793.638958:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829793.638961:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787829793.638963:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829793.638966:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1787829793.638967:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787829793.638968:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787829793.638970:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787829793.638971:0:40798:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4594 00000100:00080000:2.0:1787829793.638975:0:40798:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787829793.638977:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829793.638978:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787829793.638980:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787829793.638981:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000800:00000100:0.0:1787829793.638985:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000020:00000001:2.0:1787829793.638986:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000020:00000001:2.0:1787829793.638991:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:0.0:1787829793.638994:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1787829793.638994:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829793.638999:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:0.0:1787829793.639000:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787829793.639003:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829793.639005:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787829793.639009:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829793.639011:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4299267840 00000100:00080000:2.0:1787829793.639016:0:40798:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787829793.639019:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 00000100:00000001:2.0:1787829793.639023:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829793.639026:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4299267840 02000000:00000001:2.0:1787829793.639027:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:0.0:1787829793.639031:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 02000000:00000001:2.0:1787829793.639033:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787829793.639035:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787829793.639038:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829793.639039:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829793.639041:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1787829793.639042:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000008:00100000:2.0:1787829793.639044:0:40798:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:2.0:1787829793.639048:0:40798:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787829793.639049:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829793/real 1787829793] req@ffff8d30c9e2f480 x1874670646865280/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829818 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787829793.639050:0:40798:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829793.639057:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787829793.639060:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:0.0:1787829793.639061:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000001:2.0:1787829793.639063:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000400:00000001:0.0:1787829793.639064:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 02000000:00000001:2.0:1787829793.639065:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:0.0:1787829793.639067:0:2390:0:(events.c:78:reply_in_callback()) Process entered 02000000:00000001:2.0:1787829793.639068:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829793.639071:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787829793.639073:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829793.639074:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829793.639075:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787829793.639079:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787829793.639084:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2c000 x1874670646865536/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829793.639086:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2f480 x1874670646865280/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829818 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787829793.639111:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787829793.639112:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829793.639115:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829793.639117:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787829793.639119:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829793.639121:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:2.0:1787829793.639123:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1787829793.639124:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787829793.639127:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:2.0:1787829793.639129:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1787829793.639130:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787829793.639131:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:0.0:1787829793.639133:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787829793.639134:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1787829793.639138:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00000001:2.0:1787829793.639139:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787829793.639145:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00080000:2.0:1787829793.639146:0:40798:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4594 00000100:00000001:0.0:1787829793.639150:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00080000:2.0:1787829793.639152:0:40798:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1787829793.639155:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2f480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646865280:192.168.203.143@tcp:250:kworker.0 00000100:00000001:2.0:1787829793.639156:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829793.639159:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:0.0:1787829793.639163:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:2.0:1787829793.639163:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787829793.639167:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:0.0:1787829793.639168:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000020:00000001:2.0:1787829793.639171:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:0.0:1787829793.639172:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787829793.639176:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 00000020:00000001:2.0:1787829793.639176:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:2.0:1787829793.639180:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 02000000:00000001:0.0:1787829793.639181:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00000001:2.0:1787829793.639182:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787829793.639185:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787829793.639189:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 00000100:00080000:2.0:1787829793.639189:0:40798:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787829793.639193:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829793.639194:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787829793.639195:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787829793.639197:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 02000000:00000001:2.0:1787829793.639199:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787829793.639202:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 02000000:00000001:2.0:1787829793.639202:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829793.639205:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829793.639206:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1787829793.639208:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000008:00100000:2.0:1787829793.639212:0:40798:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00000001:0.0:1787829793.639216:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000008:00000001:2.0:1787829793.639217:0:40798:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829793.639219:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829793.639221:0:40798:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829793.639223:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787829793.639226:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787829793.639228:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1787829793.639230:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 02000000:00000001:2.0:1787829793.639230:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:0.0:1787829793.639233:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829793.639234:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829793.639235:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787829793.639238:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787829793.639239:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787829793.639241:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829793.639244:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00100000:0.0:1787829793.639246:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2da40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646865408:192.168.203.143@tcp:38:kworker.0 02000000:00000001:2.0:1787829793.639248:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829793.639250:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787829793.639252:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787829793.639254:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829793.639255:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1787829793.639258:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829793.639263:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787829793.639264:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417875008 : -126234291676608 : ffff8d30c9e2da40) 00000100:00000001:2.0:1787829793.639267:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829793.639269:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00100000:2.0:1787829793.639278:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2e940 x1874670646865664/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787829793.639286:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:0.0:1787829793.639294:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1787829793.639294:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787829793.639299:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787829793.639299:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829793.639302:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829793.639305:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1787829793.639307:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829793.639309:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829793.639314:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4299266582 00000100:00000001:0.0:1787829793.639316:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4299267840 00000100:00000001:0.0:1787829793.639318:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4299266582 00000100:00000001:0.0:1787829793.639320:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4299267840 00000100:00000001:0.0:1787829793.639322:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787829793.639324:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829793.639325:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787829793.639329:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829793/real 1787829793] req@ffff8d30c9e2da40 x1874670646865408/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829818 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787829793.639335:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787829793.639336:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787829793.639339:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787829793.639342:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787829793.639343:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829793.639344:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787829793.639345:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787829793.639348:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2da40 x1874670646865408/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829818 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787829793.639358:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829793.639359:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787829793.639361:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829793.639362:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787829793.639362:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829793.639363:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787829793.639365:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787829793.639366:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829793.639368:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787829793.639370:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787829793.639372:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2da40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646865408:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787829793.639374:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1787829793.639375:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787829793.639376:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829793.639377:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787829793.639378:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787829793.639380:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787829793.639382:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646865536:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787829793.639384:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787829793.639385:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787829793.639386:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829793.639387:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1787829793.639389:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829793.639394:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787829793.639395:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417868288 : -126234291683328 : ffff8d30c9e2c000) 00000100:00000001:0.0:1787829793.639399:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1787829793.639411:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:0.0:1787829793.639415:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1787829793.639417:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787829793.639419:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829793.639421:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787829793.639423:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829793.639427:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4299266582 00000100:00000001:0.0:1787829793.639430:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4299267840 00000100:00000001:0.0:1787829793.639433:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4299266582 00000100:00000001:0.0:1787829793.639435:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4299267840 00000100:00000001:0.0:1787829793.639437:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787829793.639439:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829793.639441:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787829793.639446:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829793/real 1787829793] req@ffff8d30c9e2c000 x1874670646865536/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829818 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787829793.639454:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787829793.639455:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787829793.639458:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787829793.639460:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787829793.639461:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829793.639463:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787829793.639464:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787829793.639468:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2c000 x1874670646865536/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829818 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787829793.639476:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829793.639478:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787829793.639481:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829793.639482:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787829793.639483:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829793.639485:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787829793.639487:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787829793.639489:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829793.639491:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787829793.639493:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787829793.639495:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2c000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646865536:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787829793.639498:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829793.639499:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787829793.639500:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787829793.639501:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787829793.639503:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787829793.639504:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787829793.639505:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787829793.639506:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787829793.639507:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787829793.639510:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787829793.639510:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829793.639512:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787829793.639512:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787829793.639513:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787829793.639514:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787829793.639515:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787829793.639516:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787829793.639517:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787829793.639517:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787829793.639519:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787829793.639520:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829793.639521:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829793.639523:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787829793.639524:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829793.639525:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787829793.639526:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787829793.639527:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1787829793.639528:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787829793.639529:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829793.639530:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787829793.639531:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787829793.639533:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787829793.639534:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646865664:192.168.203.143@tcp:8:kworker.0 00000100:00000001:0.0:1787829793.639537:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787829793.639538:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787829793.639539:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829793.639540:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1787829793.639542:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829793.639547:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787829793.639548:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417878848 : -126234291672768 : ffff8d30c9e2e940) 00000100:00000001:0.0:1787829793.639550:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1787829793.639559:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:0.0:1787829793.639561:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1787829793.639563:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787829793.639564:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829793.639565:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787829793.639567:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829793.639569:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4299266595 00000100:00000001:0.0:1787829793.639571:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4299267840 00000100:00000001:0.0:1787829793.639572:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4299266595 00000100:00000001:0.0:1787829793.639574:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4299267840 00000100:00000001:0.0:1787829793.639575:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787829793.639576:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829793.639592:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787829793.639596:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829793/real 1787829793] req@ffff8d30c9e2e940 x1874670646865664/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829818 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787829793.639621:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787829793.639622:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787829793.639625:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787829793.639626:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787829793.639627:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829793.639629:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787829793.639630:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787829793.639633:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2e940 x1874670646865664/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829818 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787829793.639638:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829793.639640:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787829793.639642:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829793.639644:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787829793.639645:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829793.639647:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787829793.639648:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787829793.639651:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829793.639654:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787829793.639656:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787829793.639658:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2e940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646865664:192.168.203.143@tcp:8:kworker.0 00000100:00000001:0.0:1787829793.639660:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829793.639661:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787829793.639662:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787829793.639664:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787829793.639665:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787829793.639666:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787829793.639667:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787829793.639669:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787829793.639669:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787829793.639672:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787829793.639673:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829793.639674:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829793.639677:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787829793.639678:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829793.639680:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787829793.639682:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829794.534686:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829794.534691:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829794.534755:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829794.534760:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829794.534764:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829795.558679:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829795.558686:0:815:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1787829795.558689:0:815:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1787829795.558691:0:815:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829795.558696:0:815:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1787829795.558697:0:815:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829795.558699:0:815:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829795.558701:0:815:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829795.558746:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829795.558749:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829795.558752:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829796.582677:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829796.582682:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829796.582770:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829796.582774:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829796.582777:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829797.606697:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829797.606703:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829797.606770:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829797.606774:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829797.606778:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829798.630669:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829798.630674:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829798.630740:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829798.630743:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829798.630746:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787829798.758745:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829798.758754:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1787829798.758756:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787829798.758760:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787829798.758765:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787829798.758771:0:40701:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 4599 00000100:00080000:0.0:1787829798.758781:0:40701:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787829798.758784:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829798.758785:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787829798.758789:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787829798.758792:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787829798.758794:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:0.0:1787829798.758797:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:0.0:1787829798.758799:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829798.758800:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:0.0:1787829798.758804:0:40701:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787829798.758806:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829798.758808:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787829798.758811:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787829798.758813:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829798.758815:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1787829798.758818:0:40701:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829798.758828:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787829798.758829:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829798.758832:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787829798.758835:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787829798.758839:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829798.758850:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787829798.758862:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d24f00 x1874670646865792/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787829798.758875:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787829798.758881:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829798.758934:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787829798.758938:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829798.758941:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1787829798.758942:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787829798.758944:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787829798.758946:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787829798.758948:0:40701:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4599 00000100:00080000:0.0:1787829798.758951:0:40701:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787829798.758953:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829798.758954:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787829798.758956:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787829798.758957:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787829798.758959:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000020:00000001:0.0:1787829798.758973:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:0.0:1787829798.758975:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829798.758976:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:0.0:1787829798.758979:0:40701:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787829798.758981:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829798.758983:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:3.0:1787829798.758986:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1787829798.758991:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 02000000:00000001:0.0:1787829798.758993:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829798.758994:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 02000000:00000001:0.0:1787829798.758998:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829798.758999:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787829798.759002:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:3.0:1787829798.759003:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000008:00100000:0.0:1787829798.759008:0:40701:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 02000000:00000001:3.0:1787829798.759009:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000008:00000001:0.0:1787829798.759013:0:40701:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829798.759013:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829798.759018:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829798.759019:0:40701:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829798.759028:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00100000:3.0:1787829798.759029:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d24f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646865792:192.168.203.143@tcp:250:kworker.0 02000000:00000001:0.0:1787829798.759030:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829798.759035:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:3.0:1787829798.759037:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787829798.759038:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787829798.759042:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829798.759043:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1787829798.759047:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829798.759048:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829798.759051:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1787829798.759067:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787829798.759068:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d24000 x1874670646865920/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787829798.759076:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787829798.759078:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829798.759081:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829798.759083:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00080000:0.0:1787829798.759084:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1787829798.759088:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601340160 : -126234108211456 : ffff8d30d4d24f00) 00000100:00080000:0.0:1787829798.759089:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1787829798.759093:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:3.0:1787829798.759094:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00080000:0.0:1787829798.759098:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787829798.759101:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787829798.759103:0:40701:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4599 00000100:00080000:0.0:1787829798.759106:0:40701:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787829798.759107:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829798.759109:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787829798.759110:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787829798.759111:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787829798.759113:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000020:00000001:0.0:1787829798.759114:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:0.0:1787829798.759116:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829798.759118:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:0.0:1787829798.759130:0:40701:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787829798.759132:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829798.759133:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787829798.759136:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787829798.759137:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829798.759139:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1787829798.759141:0:40701:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000800:00000100:3.0:1787829798.759142:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000008:00000001:0.0:1787829798.759145:0:40701:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829798.759147:0:40701:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829798.759151:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787829798.759153:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829798.759155:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:3.0:1787829798.759156:0:2390:0:(events.c:34:request_out_callback()) Process entered 02000000:00000001:0.0:1787829798.759158:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:3.0:1787829798.759161:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 02000000:00000001:0.0:1787829798.759163:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829798.759165:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829798.759168:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787829798.759171:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829798.759174:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829798.759179:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4299272960 00000100:00100000:0.0:1787829798.759183:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d270c0 x1874670646866048/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787829798.759189:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 00000100:00000001:3.0:1787829798.759195:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4299272960 00000100:00000001:3.0:1787829798.759198:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 00000100:00000001:3.0:1787829798.759211:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787829798.759214:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787829798.759214:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829798.759219:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00000001:0.0:1787829798.759221:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829798.759226:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829798/real 1787829798] req@ffff8d30d4d24f00 x1874670646865792/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829823 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787829798.759228:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787829798.759236:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829798.759244:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000400:00000001:3.0:1787829798.759244:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787829798.759247:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787829798.759250:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:3.0:1787829798.759252:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00080000:0.0:1787829798.759255:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787829798.759256:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829798.759259:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829798.759262:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829798.759264:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829798.759277:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00100000:3.0:1787829798.759281:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d24f00 x1874670646865792/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829823 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829798.759286:0:40701:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4599 00000100:00080000:0.0:1787829798.759294:0:40701:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787829798.759295:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829798.759298:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829798.759300:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787829798.759301:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:3.0:1787829798.759307:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1787829798.759308:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787829798.759311:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:3.0:1787829798.759312:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000020:00000001:0.0:1787829798.759314:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:3.0:1787829798.759315:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1787829798.759317:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:3.0:1787829798.759319:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787829798.759321:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829798.759322:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787829798.759323:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:3.0:1787829798.759328:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829798.759329:0:40701:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787829798.759333:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787829798.759334:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 02000000:00000001:0.0:1787829798.759336:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:3.0:1787829798.759338:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 02000000:00000001:0.0:1787829798.759340:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00100000:3.0:1787829798.759344:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d24f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646865792:192.168.203.143@tcp:250:kworker.0 02000000:00000001:0.0:1787829798.759345:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829798.759350:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:3.0:1787829798.759353:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000008:00100000:0.0:1787829798.759354:0:40701:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00000001:3.0:1787829798.759357:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000008:00000001:0.0:1787829798.759358:0:40701:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829798.759360:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 00000100:00000001:0.0:1787829798.759361:0:40701:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829798.759363:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787829798.759367:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787829798.759370:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829798.759371:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787829798.759372:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787829798.759374:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829798.759377:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00000001:0.0:1787829798.759379:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829798.759392:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829798.759395:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 00000100:00100000:0.0:1787829798.759397:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d25a40 x1874670646866176/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:3.0:1787829798.759400:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829798.759404:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829798.759408:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787829798.759409:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787829798.759412:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829798.759413:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829798.759416:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829798.759418:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829798.759424:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787829798.759427:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829798.759429:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1787829798.759431:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787829798.759432:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787829798.759434:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787829798.759436:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829798.759438:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787829798.759441:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829798.759442:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829798.759446:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d24000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646865920:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829798.759448:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1787829798.759449:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1787829798.759456:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829798.759458:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1787829798.759460:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829798.759464:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787829798.759466:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601336320 : -126234108215296 : ffff8d30d4d24000) 00000100:00000001:3.0:1787829798.759470:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1787829798.759479:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:3.0:1787829798.759482:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787829798.759484:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829798.759485:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829798.759486:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1787829798.759488:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829798.759492:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4299271702 00000100:00000001:3.0:1787829798.759494:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4299272960 00000100:00000001:3.0:1787829798.759496:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4299271702 00000100:00000001:3.0:1787829798.759498:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4299272960 00000100:00000001:3.0:1787829798.759499:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829798.759500:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829798.759502:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1787829798.759505:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829798/real 1787829798] req@ffff8d30d4d24000 x1874670646865920/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829823 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1787829798.759510:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787829798.759511:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829798.759512:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829798.759514:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829798.759515:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829798.759516:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829798.759517:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829798.759520:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d24000 x1874670646865920/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829823 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829798.759525:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829798.759527:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829798.759529:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829798.759530:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829798.759530:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829798.759532:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829798.759533:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829798.759534:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829798.759536:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829798.759538:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829798.759540:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d24000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646865920:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829798.759737:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787829798.759740:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787829798.759742:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829798.759746:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787829798.759748:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829798.759751:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829798.759756:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d270c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646866048:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829798.759761:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1787829798.759762:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1787829798.759764:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829798.759765:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1787829798.759768:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829798.759773:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787829798.759774:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601348800 : -126234108202816 : ffff8d30d4d270c0) 00000100:00000001:3.0:1787829798.759778:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1787829798.759789:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:3.0:1787829798.759794:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787829798.759796:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829798.759798:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829798.759799:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1787829798.759801:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829798.759804:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4299271703 00000100:00000001:3.0:1787829798.759806:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4299272960 00000100:00000001:3.0:1787829798.759808:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4299271703 00000100:00000001:3.0:1787829798.759809:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4299272960 00000100:00000001:3.0:1787829798.759811:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829798.759812:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829798.759814:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1787829798.759817:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829798/real 1787829798] req@ffff8d30d4d270c0 x1874670646866048/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829823 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1787829798.759823:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787829798.759824:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829798.759825:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829798.759827:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829798.759828:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829798.759829:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829798.759830:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829798.759833:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d270c0 x1874670646866048/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829823 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829798.759838:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829798.759839:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829798.759841:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829798.759842:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829798.759843:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829798.759845:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829798.759846:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829798.759848:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829798.759850:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829798.759852:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829798.759855:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d270c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646866048:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829798.759858:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787829798.759860:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787829798.759861:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829798.759863:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787829798.759864:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829798.759865:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829798.759867:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d25a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646866176:192.168.203.143@tcp:8:kworker.0 00000100:00000001:3.0:1787829798.759870:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1787829798.759871:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1787829798.759883:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829798.759884:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1787829798.759886:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829798.759890:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787829798.759891:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601343040 : -126234108208576 : ffff8d30d4d25a40) 00000100:00000001:3.0:1787829798.759894:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1787829798.759900:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:3.0:1787829798.759904:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787829798.759922:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829798.759923:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829798.759924:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1787829798.759926:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829798.759929:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4299271716 00000100:00000001:3.0:1787829798.759931:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4299272960 00000100:00000001:3.0:1787829798.759932:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4299271716 00000100:00000001:3.0:1787829798.759934:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4299272960 00000100:00000001:3.0:1787829798.759935:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829798.759937:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829798.759938:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1787829798.759940:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829798/real 1787829798] req@ffff8d30d4d25a40 x1874670646866176/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829823 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1787829798.759957:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787829798.759958:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829798.759959:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829798.759960:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829798.759961:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829798.759962:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829798.759963:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829798.759965:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d25a40 x1874670646866176/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829823 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829798.759970:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829798.759971:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829798.759972:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829798.759973:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829798.759974:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829798.759976:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829798.759977:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829798.759979:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829798.759981:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829798.759982:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829798.759984:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d25a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646866176:192.168.203.143@tcp:8:kworker.0 00000100:00000001:3.0:1787829798.759986:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829798.759987:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829798.759988:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829798.759989:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829798.759991:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829798.759992:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829798.759993:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829798.759994:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829798.759995:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829798.759996:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829798.759997:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829798.759999:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829798.760000:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829798.760000:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829798.760002:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829798.760003:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829798.760004:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829798.760005:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829798.760006:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829798.760008:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829798.760009:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829798.760010:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829798.760011:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829798.760011:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829798.760012:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829798.760013:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829798.760014:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829798.760015:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829798.760015:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829798.760017:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829798.760017:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829798.760019:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829798.760021:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787829798.760022:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829798.760023:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1787829798.760024:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829799.654678:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829799.654684:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829799.654718:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829799.654722:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829799.654724:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829800.678654:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829800.678660:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829800.678708:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829800.678711:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829800.678714:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829801.702670:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829801.702674:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829801.702715:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829801.702718:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829801.702721:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829802.726652:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829802.726657:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829802.726712:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829802.726716:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829802.726720:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829803.750669:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829803.750677:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829803.750728:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829803.750731:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829803.750734:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787829803.878666:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829803.878674:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1787829803.878676:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787829803.878679:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787829803.878683:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787829803.878689:0:40798:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 4604 00000100:00080000:0.0:1787829803.878697:0:40798:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787829803.878699:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829803.878701:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787829803.878704:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787829803.878714:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787829803.878716:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:0.0:1787829803.878718:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:0.0:1787829803.878722:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829803.878723:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:0.0:1787829803.878727:0:40798:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787829803.878729:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829803.878731:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787829803.878734:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787829803.878736:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829803.878737:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1787829803.878739:0:40798:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829803.878750:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787829803.878751:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829803.878753:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787829803.878755:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787829803.878759:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829803.878767:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787829803.878775:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d252c0 x1874670646866304/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787829803.878782:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787829803.878786:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829803.878799:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787829803.878801:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829803.878803:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1787829803.878804:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787829803.878805:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787829803.878807:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787829803.878809:0:40798:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4604 00000100:00080000:0.0:1787829803.878811:0:40798:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787829803.878812:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829803.878813:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787829803.878815:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787829803.878816:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787829803.878817:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000020:00000001:0.0:1787829803.878819:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:0.0:1787829803.878820:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829803.878821:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:0.0:1787829803.878823:0:40798:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787829803.878824:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829803.878825:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787829803.878829:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787829803.878831:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829803.878833:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1787829803.878837:0:40798:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1787829803.878839:0:40798:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829803.878840:0:40798:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829803.878842:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787829803.878843:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829803.878846:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:3.0:1787829803.878846:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:0.0:1787829803.878847:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787829803.878849:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829803.878854:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829803.878854:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787829803.878859:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00100000:0.0:1787829803.878860:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d25680 x1874670646866432/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787829803.878864:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787829803.878869:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829803.878870:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787829803.878874:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829803.878876:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:0.0:1787829803.878877:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787829803.878881:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 02000000:00000001:3.0:1787829803.878881:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00080000:0.0:1787829803.878888:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 02000000:00000001:3.0:1787829803.878888:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829803.878892:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787829803.878894:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787829803.878897:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00100000:3.0:1787829803.878899:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d252c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646866304:192.168.203.143@tcp:250:kworker.0 00000100:00080000:0.0:1787829803.878901:0:40798:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4604 00000100:00080000:0.0:1787829803.878906:0:40798:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787829803.878908:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:0.0:1787829803.878909:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829803.878913:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787829803.878914:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787829803.878916:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 02000000:00000001:3.0:1787829803.878916:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000020:00000001:0.0:1787829803.878920:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 02000000:00000001:3.0:1787829803.878922:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787829803.878923:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:0.0:1787829803.878928:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 02000000:00000001:3.0:1787829803.878929:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:0.0:1787829803.878930:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:0.0:1787829803.878934:0:40798:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787829803.878936:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829803.878936:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829803.878937:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787829803.878941:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787829803.878944:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829803.878945:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1787829803.878946:0:40798:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1787829803.878948:0:40798:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829803.878949:0:40798:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829803.878953:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787829803.878954:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829803.878955:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787829803.878956:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787829803.878960:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:3.0:1787829803.878960:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601341120 : -126234108210496 : ffff8d30d4d252c0) 02000000:00000001:0.0:1787829803.878963:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829803.878968:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829803.878968:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00100000:0.0:1787829803.878974:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d27c00 x1874670646866560/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787829803.878979:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787829803.878980:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829803.878982:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787829803.878984:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829803.878986:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1787829803.878987:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787829803.878988:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787829803.878989:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787829803.878991:0:40798:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4604 00000100:00080000:0.0:1787829803.878993:0:40798:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787829803.878994:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829803.878995:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787829803.878997:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787829803.878998:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787829803.878999:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000020:00000001:0.0:1787829803.879001:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:0.0:1787829803.879002:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829803.879003:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:0.0:1787829803.879004:0:40798:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787829803.879006:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829803.879007:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787829803.879009:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787829803.879015:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:3.0:1787829803.879015:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:0.0:1787829803.879019:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1787829803.879021:0:40798:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1787829803.879023:0:40798:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829803.879024:0:40798:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829803.879028:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787829803.879029:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829803.879030:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787829803.879031:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:3.0:1787829803.879032:0:2390:0:(events.c:34:request_out_callback()) Process entered 02000000:00000001:0.0:1787829803.879033:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829803.879039:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787829803.879040:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829803.879044:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787829803.879046:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d26940 x1874670646866688/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787829803.879049:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787829803.879056:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787829803.879058:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829803.879058:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829803.879064:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829803.879067:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4299278080 00000100:00000001:3.0:1787829803.879078:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 00000100:00000001:3.0:1787829803.879085:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4299278080 00000100:00000001:3.0:1787829803.879089:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 00000100:00000001:3.0:1787829803.879093:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829803.879095:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829803.879098:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1787829803.879105:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829803/real 1787829803] req@ffff8d30d4d252c0 x1874670646866304/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829828 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1787829803.879117:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787829803.879119:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829803.879122:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829803.879125:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829803.879127:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829803.879129:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829803.879131:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829803.879135:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d252c0 x1874670646866304/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829828 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829803.879144:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829803.879147:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829803.879151:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829803.879152:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829803.879154:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829803.879157:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829803.879159:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829803.879162:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829803.879166:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829803.879168:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829803.879172:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d252c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646866304:192.168.203.143@tcp:250:kworker.0 00000100:00000001:3.0:1787829803.879178:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829803.879180:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829803.879182:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829803.879184:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829803.879188:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829803.879190:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829803.879192:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829803.879196:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829803.879198:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829803.879201:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829803.879202:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829803.879205:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829803.879211:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787829803.879213:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829803.879215:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1787829803.879217:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787829803.879218:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787829803.879220:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787829803.879221:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829803.879224:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787829803.879226:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829803.879228:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829803.879231:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d25680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646866432:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829803.879235:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1787829803.879237:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1787829803.879239:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829803.879241:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1787829803.879244:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829803.879251:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787829803.879252:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601342080 : -126234108209536 : ffff8d30d4d25680) 00000100:00000001:3.0:1787829803.879257:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1787829803.879271:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:3.0:1787829803.879276:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787829803.879279:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829803.879281:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829803.879283:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1787829803.879286:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829803.879290:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4299276822 00000100:00000001:3.0:1787829803.879293:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4299278080 00000100:00000001:3.0:1787829803.879295:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4299276822 00000100:00000001:3.0:1787829803.879298:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4299278080 00000100:00000001:3.0:1787829803.879300:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829803.879302:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829803.879304:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1787829803.879309:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829803/real 1787829803] req@ffff8d30d4d25680 x1874670646866432/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829828 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1787829803.879318:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787829803.879320:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829803.879323:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829803.879324:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829803.879326:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829803.879328:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829803.879329:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829803.879334:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d25680 x1874670646866432/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829828 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829803.879341:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829803.879344:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829803.879346:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829803.879348:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829803.879349:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829803.879352:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829803.879354:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829803.879357:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829803.879360:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829803.879362:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829803.879366:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d25680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646866432:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829803.879370:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787829803.879372:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787829803.879373:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829803.879376:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787829803.879378:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829803.879380:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829803.879383:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d27c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646866560:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829803.879386:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1787829803.879388:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1787829803.879390:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829803.879391:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1787829803.879395:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829803.879400:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787829803.879401:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601351680 : -126234108199936 : ffff8d30d4d27c00) 00000100:00000001:3.0:1787829803.879404:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1787829803.879417:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:3.0:1787829803.879422:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787829803.879424:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829803.879426:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829803.879428:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1787829803.879431:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829803.879434:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4299276822 00000100:00000001:3.0:1787829803.879437:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4299278080 00000100:00000001:3.0:1787829803.879439:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4299276822 00000100:00000001:3.0:1787829803.879442:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4299278080 00000100:00000001:3.0:1787829803.879445:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829803.879447:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829803.879449:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1787829803.879453:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829803/real 1787829803] req@ffff8d30d4d27c00 x1874670646866560/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829828 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1787829803.879461:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787829803.879462:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829803.879465:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829803.879466:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829803.879467:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829803.879469:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829803.879471:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829803.879475:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d27c00 x1874670646866560/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829828 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829803.879482:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829803.879485:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829803.879487:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829803.879489:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829803.879490:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829803.879492:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829803.879494:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829803.879496:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829803.879499:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829803.879502:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829803.879505:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d27c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646866560:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829803.879527:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787829803.879529:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787829803.879530:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829803.879532:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787829803.879534:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829803.879537:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829803.879541:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d26940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646866688:192.168.203.143@tcp:8:kworker.0 00000100:00000001:3.0:1787829803.879544:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1787829803.879546:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1787829803.879548:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829803.879550:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1787829803.879553:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829803.879559:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787829803.879561:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601346880 : -126234108204736 : ffff8d30d4d26940) 00000100:00000001:3.0:1787829803.879565:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1787829803.879578:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:3.0:1787829803.879584:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787829803.879587:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829803.879588:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829803.879590:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1787829803.879593:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829803.879596:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4299276835 00000100:00000001:3.0:1787829803.879632:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4299278080 00000100:00000001:3.0:1787829803.879636:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4299276836 00000100:00000001:3.0:1787829803.879638:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4299278080 00000100:00000001:3.0:1787829803.879641:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829803.879643:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829803.879645:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1787829803.879650:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829803/real 1787829803] req@ffff8d30d4d26940 x1874670646866688/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829828 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1787829803.879659:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787829803.879660:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829803.879664:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829803.879666:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829803.879667:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829803.879669:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829803.879671:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829803.879675:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d26940 x1874670646866688/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829828 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829803.879689:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829803.879691:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829803.879693:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829803.879695:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829803.879697:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829803.879699:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829803.879700:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829803.879703:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829803.879706:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829803.879708:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829803.879712:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d26940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646866688:192.168.203.143@tcp:8:kworker.0 00000100:00000001:3.0:1787829803.879715:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829803.879718:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829803.879719:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829803.879720:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829803.879723:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829803.879725:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829803.879726:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829803.879728:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829803.879730:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829803.879732:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829803.879733:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829803.879735:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829803.879736:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829803.879738:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829803.879740:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829803.879741:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829803.879742:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829803.879744:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829803.879745:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829803.879747:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829803.879749:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829803.879750:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829803.879752:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829803.879753:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829803.879755:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829803.879757:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829803.879758:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829803.879759:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829803.879760:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829803.879763:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829803.879764:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829803.879766:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829803.879770:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787829803.879772:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829803.879774:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1787829803.879776:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829804.774654:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829804.774662:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829804.774708:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829804.774711:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829804.774714:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829805.798713:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829805.798720:0:815:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1787829805.798722:0:815:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1787829805.798724:0:815:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829805.798728:0:815:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1787829805.798729:0:815:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829805.798731:0:815:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829805.798733:0:815:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829805.798793:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829805.798797:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829805.798801:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829806.822729:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829806.822737:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829806.822789:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829806.822794:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829806.822797:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829807.846743:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829807.846749:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829807.846818:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829807.846822:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829807.846826:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829808.870694:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829808.870699:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829808.870755:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829808.870758:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829808.870761:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787829808.998672:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829808.998680:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1787829808.998682:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787829808.998686:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787829808.998708:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787829808.998714:0:40701:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 4609 00000100:00080000:0.0:1787829808.998724:0:40701:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787829808.998727:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829808.998728:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787829808.998732:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787829808.998734:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787829808.998737:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:0.0:1787829808.998739:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:0.0:1787829808.998748:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829808.998750:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:0.0:1787829808.998753:0:40701:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787829808.998756:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829808.998759:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787829808.998762:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787829808.998764:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829808.998766:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1787829808.998768:0:40701:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829808.998779:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787829808.998780:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829808.998783:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787829808.998785:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787829808.998789:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829808.998800:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787829808.998808:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d26940 x1874670646866816/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787829808.998817:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787829808.998821:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829808.998835:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787829808.998838:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829808.998840:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1787829808.998842:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787829808.998843:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787829808.998846:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787829808.998847:0:40701:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4609 00000100:00080000:0.0:1787829808.998850:0:40701:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787829808.998851:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829808.998853:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787829808.998854:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787829808.998855:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787829808.998857:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000020:00000001:0.0:1787829808.998859:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:0.0:1787829808.998860:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829808.998861:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:0.0:1787829808.998863:0:40701:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787829808.998866:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829808.998867:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787829808.998872:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787829808.998875:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829808.998877:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1787829808.998882:0:40701:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1787829808.998886:0:40701:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829808.998888:0:40701:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829808.998893:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787829808.998895:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829808.998898:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787829808.998900:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787829808.998902:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829808.998910:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829808.998911:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1787829808.998917:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00100000:0.0:1787829808.998919:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d27c00 x1874670646866944/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787829808.998920:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787829808.998924:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787829808.998927:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829808.998934:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787829808.998938:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787829808.998941:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 02000000:00000001:3.0:1787829808.998942:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829808.998946:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829808.998951:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d26940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646866816:192.168.203.143@tcp:250:kworker.0 00000100:00000001:0.0:1787829808.998952:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787829808.998958:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1787829808.998959:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00080000:0.0:1787829808.998964:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 02000000:00000001:3.0:1787829808.998965:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:0.0:1787829808.998966:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787829808.998971:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 02000000:00000001:3.0:1787829808.998971:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829808.998976:0:40701:0:(import.c:485:import_select_connection()) Process entered 02000000:00000001:3.0:1787829808.998977:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00080000:0.0:1787829808.998982:0:40701:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4609 02000000:00000001:3.0:1787829808.998984:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787829808.998987:0:40701:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787829808.998990:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829808.998992:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787829808.998994:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787829808.998996:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787829808.998998:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000020:00000001:0.0:1787829808.999000:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:3.0:1787829808.999000:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787829808.999005:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829808.999005:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601346880 : -126234108204736 : ffff8d30d4d26940) 00000100:00000001:0.0:1787829808.999009:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:3.0:1787829808.999011:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00080000:0.0:1787829808.999015:0:40701:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787829808.999018:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829808.999019:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787829808.999022:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787829808.999024:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829808.999026:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1787829808.999028:0:40701:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1787829808.999030:0:40701:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829808.999032:0:40701:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829808.999036:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787829808.999038:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829808.999040:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787829808.999041:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787829808.999042:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829808.999048:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:3.0:1787829808.999052:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00100000:0.0:1787829808.999057:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d25680 x1874670646867072/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787829808.999065:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787829808.999067:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829808.999068:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1787829808.999070:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787829808.999074:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1787829808.999075:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00080000:0.0:1787829808.999080:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1787829808.999081:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829808.999083:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:3.0:1787829808.999084:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00080000:0.0:1787829808.999086:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787829808.999090:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:3.0:1787829808.999091:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787829808.999093:0:40701:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4609 00000100:00000001:3.0:1787829808.999098:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4299283200 00000100:00080000:0.0:1787829808.999099:0:40701:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787829808.999103:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829808.999104:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787829808.999107:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787829808.999107:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787829808.999109:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:3.0:1787829808.999110:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 00000020:00000001:0.0:1787829808.999112:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:0.0:1787829808.999115:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829808.999117:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:3.0:1787829808.999119:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4299283200 00000100:00080000:0.0:1787829808.999120:0:40701:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787829808.999126:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829808.999128:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 02000000:00000001:0.0:1787829808.999129:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:3.0:1787829808.999133:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787829808.999134:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829808.999140:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829808.999141:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829808.999147:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:3.0:1787829808.999148:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000008:00100000:0.0:1787829808.999149:0:40701:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1787829808.999153:0:40701:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829808.999155:0:40701:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829808.999159:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00100000:3.0:1787829808.999160:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829808/real 1787829808] req@ffff8d30d4d26940 x1874670646866816/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829833 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:0.0:1787829808.999161:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829808.999174:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787829808.999178:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787829808.999181:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:3.0:1787829808.999181:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000001:0.0:1787829808.999184:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:3.0:1787829808.999186:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00100000:0.0:1787829808.999192:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d252c0 x1874670646867200/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787829808.999193:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829808.999199:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787829808.999202:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787829808.999204:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829808.999206:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829808.999211:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829808.999212:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829808.999216:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829808.999225:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d26940 x1874670646866816/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829833 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829808.999235:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829808.999239:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829808.999241:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829808.999243:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829808.999244:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829808.999246:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829808.999248:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829808.999250:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829808.999254:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829808.999256:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829808.999260:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d26940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646866816:192.168.203.143@tcp:250:kworker.0 00000100:00000001:3.0:1787829808.999265:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829808.999267:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829808.999268:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829808.999270:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829808.999273:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829808.999274:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829808.999276:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829808.999279:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829808.999281:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829808.999284:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829808.999285:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829808.999287:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829808.999294:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787829808.999295:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829808.999297:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1787829808.999298:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787829808.999299:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787829808.999301:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787829808.999302:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829808.999304:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787829808.999306:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829808.999307:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829808.999310:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d27c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646866944:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829808.999313:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1787829808.999314:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1787829808.999316:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829808.999317:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1787829808.999320:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829808.999325:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787829808.999327:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601351680 : -126234108199936 : ffff8d30d4d27c00) 00000100:00000001:3.0:1787829808.999331:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1787829808.999342:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:3.0:1787829808.999347:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787829808.999349:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829808.999350:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829808.999351:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1787829808.999354:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829808.999357:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4299281942 00000100:00000001:3.0:1787829808.999376:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4299283200 00000100:00000001:3.0:1787829808.999378:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4299281942 00000100:00000001:3.0:1787829808.999379:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4299283200 00000100:00000001:3.0:1787829808.999381:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829808.999382:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829808.999383:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1787829808.999386:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829808/real 1787829808] req@ffff8d30d4d27c00 x1874670646866944/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829833 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1787829808.999391:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787829808.999392:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829808.999394:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829808.999395:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829808.999396:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829808.999397:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829808.999398:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829808.999400:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d27c00 x1874670646866944/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829833 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829808.999405:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829808.999407:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829808.999408:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829808.999409:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829808.999410:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829808.999411:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829808.999412:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829808.999413:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829808.999415:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829808.999416:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829808.999419:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d27c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646866944:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829808.999422:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787829808.999423:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787829808.999424:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829808.999425:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787829808.999426:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829808.999427:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829808.999429:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d25680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646867072:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829808.999430:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1787829808.999432:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1787829808.999433:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829808.999434:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1787829808.999435:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829808.999438:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787829808.999439:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601342080 : -126234108209536 : ffff8d30d4d25680) 00000100:00000001:3.0:1787829808.999441:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1787829808.999448:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:3.0:1787829808.999451:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787829808.999452:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829808.999453:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829808.999454:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1787829808.999456:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829808.999458:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4299281942 00000100:00000001:3.0:1787829808.999460:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4299283200 00000100:00000001:3.0:1787829808.999461:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4299281942 00000100:00000001:3.0:1787829808.999462:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4299283200 00000100:00000001:3.0:1787829808.999464:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829808.999465:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829808.999466:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1787829808.999468:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829808/real 1787829808] req@ffff8d30d4d25680 x1874670646867072/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829833 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1787829808.999496:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787829808.999497:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829808.999499:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829808.999500:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829808.999501:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829808.999502:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829808.999503:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829808.999507:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d25680 x1874670646867072/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829833 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829808.999511:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829808.999513:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829808.999514:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829808.999515:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829808.999516:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829808.999517:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829808.999518:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829808.999520:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829808.999521:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829808.999522:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829808.999524:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d25680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646867072:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829808.999527:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787829808.999527:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787829808.999528:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829808.999530:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787829808.999531:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829808.999532:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829808.999534:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d252c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646867200:192.168.203.143@tcp:8:kworker.0 00000100:00000001:3.0:1787829808.999536:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1787829808.999537:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1787829808.999539:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829808.999540:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1787829808.999542:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829808.999546:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787829808.999547:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601341120 : -126234108210496 : ffff8d30d4d252c0) 00000100:00000001:3.0:1787829808.999550:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1787829808.999558:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:3.0:1787829808.999560:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787829808.999562:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829808.999563:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829808.999564:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1787829808.999565:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829808.999567:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4299281955 00000100:00000001:3.0:1787829808.999569:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4299283200 00000100:00000001:3.0:1787829808.999571:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4299281955 00000100:00000001:3.0:1787829808.999572:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4299283200 00000100:00000001:3.0:1787829808.999574:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829808.999575:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829808.999576:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1787829808.999578:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829808/real 1787829808] req@ffff8d30d4d252c0 x1874670646867200/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829833 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1787829808.999583:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787829808.999584:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829808.999585:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829808.999586:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829808.999586:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829808.999587:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829808.999588:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829808.999590:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d252c0 x1874670646867200/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829833 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829808.999595:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829808.999596:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829808.999598:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829808.999599:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829808.999630:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829808.999631:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829808.999633:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829808.999634:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829808.999637:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829808.999638:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829808.999640:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d252c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646867200:192.168.203.143@tcp:8:kworker.0 00000100:00000001:3.0:1787829808.999642:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829808.999644:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829808.999645:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829808.999646:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829808.999647:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829808.999648:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829808.999649:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829808.999651:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829808.999652:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829808.999655:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829808.999655:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829808.999657:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829808.999658:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829808.999658:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829808.999660:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829808.999660:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829808.999661:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829808.999662:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829808.999663:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829808.999664:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829808.999665:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829808.999666:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829808.999666:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829808.999667:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829808.999668:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829808.999668:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829808.999669:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829808.999670:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829808.999671:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829808.999672:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829808.999673:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829808.999674:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829808.999677:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787829808.999678:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829808.999679:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1787829808.999680:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829809.894677:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829809.894682:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829809.894751:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829809.894756:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829809.894759:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829810.918679:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829810.918684:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829810.918736:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829810.918739:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829810.918742:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829811.942715:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829811.942723:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829811.942806:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829811.942810:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829811.942812:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829812.966707:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829812.966721:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829812.966787:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829812.966791:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829812.966795:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829813.990697:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829813.990704:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829813.990760:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829813.990765:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829813.990769:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787829814.118702:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829814.118710:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1787829814.118712:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787829814.118716:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787829814.118719:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787829814.118724:0:40701:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 4614 00000100:00080000:0.0:1787829814.118734:0:40701:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787829814.118737:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829814.118739:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787829814.118742:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787829814.118744:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787829814.118747:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:0.0:1787829814.118749:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:0.0:1787829814.118751:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829814.118752:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:0.0:1787829814.118756:0:40701:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787829814.118758:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829814.118760:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787829814.118763:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787829814.118765:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829814.118767:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1787829814.118769:0:40701:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829814.118779:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787829814.118781:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829814.118784:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787829814.118786:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787829814.118790:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829814.118799:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787829814.118806:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d25a40 x1874670646867328/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787829814.118814:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787829814.118817:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829814.118826:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787829814.118828:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829814.118831:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1787829814.118832:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787829814.118833:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787829814.118835:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787829814.118837:0:40701:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4614 00000100:00080000:0.0:1787829814.118840:0:40701:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787829814.118841:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829814.118843:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787829814.118844:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787829814.118846:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787829814.118847:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000020:00000001:0.0:1787829814.118848:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:0.0:1787829814.118850:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829814.118851:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:0.0:1787829814.118853:0:40701:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787829814.118855:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829814.118856:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787829814.118861:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787829814.118863:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829814.118864:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:3.0:1787829814.118866:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000008:00100000:0.0:1787829814.118868:0:40701:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1787829814.118872:0:40701:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829814.118874:0:40701:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829814.118874:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787829814.118879:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 02000000:00000001:0.0:1787829814.118881:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:3.0:1787829814.118883:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 02000000:00000001:0.0:1787829814.118885:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829814.118885:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829814.118892:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787829814.118892:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787829814.118895:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829814.118897:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829814.118902:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d25a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646867328:192.168.203.143@tcp:250:kworker.0 02000000:00000001:0.0:1787829814.118903:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787829814.118908:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829814.118909:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1787829814.118915:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:0.0:1787829814.118918:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829814.118920:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829814.118925:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00100000:0.0:1787829814.118928:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d270c0 x1874670646867456/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:3.0:1787829814.118931:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829814.118942:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787829814.118944:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829814.118948:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787829814.118951:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829814.118954:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1787829814.118957:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:3.0:1787829814.118958:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00080000:0.0:1787829814.118959:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787829814.118963:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601343040 : -126234108208576 : ffff8d30d4d25a40) 00000100:00000001:0.0:1787829814.118964:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787829814.118969:0:40701:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4614 00000100:00000001:3.0:1787829814.118971:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00080000:0.0:1787829814.118975:0:40701:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787829814.118979:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829814.118981:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787829814.118984:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787829814.118985:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787829814.118986:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000020:00000001:0.0:1787829814.118988:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:0.0:1787829814.118990:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829814.118991:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:0.0:1787829814.118994:0:40701:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787829814.118995:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829814.118996:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787829814.118999:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787829814.119000:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829814.119002:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1787829814.119003:0:40701:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1787829814.119005:0:40701:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829814.119006:0:40701:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829814.119010:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787829814.119011:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829814.119012:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787829814.119013:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787829814.119014:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829814.119018:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:3.0:1787829814.119019:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00100000:0.0:1787829814.119024:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d24000 x1874670646867584/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787829814.119030:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787829814.119032:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1787829814.119033:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829814.119037:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829814.119038:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00080000:0.0:1787829814.119040:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:3.0:1787829814.119042:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829814.119046:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00080000:0.0:1787829814.119047:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1787829814.119050:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787829814.119052:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787829814.119052:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829814.119059:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:3.0:1787829814.119060:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4299288320 00000100:00080000:0.0:1787829814.119062:0:40701:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4614 00000100:00080000:0.0:1787829814.119067:0:40701:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787829814.119069:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829814.119070:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:3.0:1787829814.119071:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 00000020:00000001:0.0:1787829814.119074:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787829814.119076:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787829814.119078:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:3.0:1787829814.119078:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4299288320 00000020:00000001:0.0:1787829814.119084:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:3.0:1787829814.119085:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1787829814.119086:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829814.119089:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:3.0:1787829814.119090:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00080000:0.0:1787829814.119095:0:40701:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:3.0:1787829814.119096:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829814.119101:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829814.119102:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 02000000:00000001:0.0:1787829814.119106:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787829814.119109:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00100000:3.0:1787829814.119110:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829814/real 1787829814] req@ffff8d30d4d25a40 x1874670646867328/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829839 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:0.0:1787829814.119112:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829814.119115:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1787829814.119118:0:40701:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1787829814.119122:0:40701:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829814.119126:0:40701:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:3.0:1787829814.119129:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 02000000:00000001:0.0:1787829814.119131:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000400:00000001:3.0:1787829814.119132:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 02000000:00000001:0.0:1787829814.119134:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829814.119136:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787829814.119137:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787829814.119140:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:3.0:1787829814.119141:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 02000000:00000001:0.0:1787829814.119142:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829814.119144:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829814.119147:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829814.119148:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829814.119150:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787829814.119155:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d24f00 x1874670646867712/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1787829814.119159:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d25a40 x1874670646867328/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829839 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787829814.119166:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787829814.119169:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829814.119173:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829814.119182:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829814.119185:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829814.119189:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829814.119191:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829814.119193:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829814.119196:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829814.119198:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829814.119202:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829814.119207:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829814.119209:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829814.119212:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d25a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646867328:192.168.203.143@tcp:250:kworker.0 00000100:00000001:3.0:1787829814.119215:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829814.119218:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829814.119220:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829814.119222:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829814.119225:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829814.119226:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829814.119227:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829814.119231:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829814.119232:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829814.119236:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829814.119237:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829814.119239:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829814.119247:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787829814.119249:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829814.119251:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1787829814.119253:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787829814.119254:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787829814.119256:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787829814.119257:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829814.119259:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787829814.119261:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829814.119264:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829814.119268:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d270c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646867456:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829814.119272:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1787829814.119274:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1787829814.119275:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829814.119277:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1787829814.119280:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829814.119284:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787829814.119285:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601348800 : -126234108202816 : ffff8d30d4d270c0) 00000100:00000001:3.0:1787829814.119289:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1787829814.119299:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:3.0:1787829814.119303:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787829814.119305:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829814.119306:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829814.119307:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1787829814.119309:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829814.119312:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4299287062 00000100:00000001:3.0:1787829814.119314:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4299288320 00000100:00000001:3.0:1787829814.119316:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4299287062 00000100:00000001:3.0:1787829814.119318:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4299288320 00000100:00000001:3.0:1787829814.119319:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829814.119321:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829814.119322:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1787829814.119325:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829814/real 1787829814] req@ffff8d30d4d270c0 x1874670646867456/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829839 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1787829814.119331:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787829814.119332:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829814.119334:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829814.119335:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829814.119336:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829814.119337:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829814.119338:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829814.119341:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d270c0 x1874670646867456/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829839 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829814.119345:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829814.119347:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829814.119348:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829814.119349:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829814.119350:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829814.119351:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829814.119352:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829814.119354:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829814.119356:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829814.119357:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829814.119359:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d270c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646867456:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829814.119361:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787829814.119362:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787829814.119363:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829814.119365:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787829814.119366:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829814.119368:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829814.119370:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d24000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646867584:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829814.119373:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1787829814.119375:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1787829814.119376:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829814.119378:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1787829814.119381:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829814.119386:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787829814.119387:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601336320 : -126234108215296 : ffff8d30d4d24000) 00000100:00000001:3.0:1787829814.119391:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1787829814.119403:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:3.0:1787829814.119407:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787829814.119409:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829814.119411:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829814.119413:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1787829814.119415:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829814.119428:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4299287062 00000100:00000001:3.0:1787829814.119430:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4299288320 00000100:00000001:3.0:1787829814.119432:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4299287062 00000100:00000001:3.0:1787829814.119433:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4299288320 00000100:00000001:3.0:1787829814.119434:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829814.119449:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829814.119450:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1787829814.119454:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829814/real 1787829814] req@ffff8d30d4d24000 x1874670646867584/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829839 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1787829814.119459:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787829814.119460:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829814.119462:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829814.119463:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829814.119464:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829814.119466:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829814.119467:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829814.119469:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d24000 x1874670646867584/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829839 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829814.119474:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829814.119476:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829814.119477:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829814.119478:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829814.119479:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829814.119480:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829814.119481:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829814.119482:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829814.119484:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829814.119485:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829814.119492:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d24000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646867584:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829814.119495:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787829814.119496:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787829814.119497:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829814.119499:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787829814.119500:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829814.119501:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829814.119503:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d24f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646867712:192.168.203.143@tcp:8:kworker.0 00000100:00000001:3.0:1787829814.119505:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1787829814.119507:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1787829814.119508:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829814.119509:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1787829814.119511:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829814.119514:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787829814.119515:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601340160 : -126234108211456 : ffff8d30d4d24f00) 00000100:00000001:3.0:1787829814.119518:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1787829814.119526:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:3.0:1787829814.119529:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787829814.119531:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829814.119532:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829814.119533:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1787829814.119535:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829814.119537:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4299287075 00000100:00000001:3.0:1787829814.119539:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4299288320 00000100:00000001:3.0:1787829814.119540:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4299287075 00000100:00000001:3.0:1787829814.119542:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4299288320 00000100:00000001:3.0:1787829814.119543:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829814.119544:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829814.119545:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1787829814.119548:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829814/real 1787829814] req@ffff8d30d4d24f00 x1874670646867712/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829839 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1787829814.119552:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787829814.119553:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829814.119554:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829814.119555:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829814.119556:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829814.119557:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829814.119558:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829814.119560:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d24f00 x1874670646867712/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829839 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829814.119564:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829814.119566:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829814.119567:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829814.119568:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829814.119569:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829814.119570:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829814.119571:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829814.119572:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829814.119574:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829814.119576:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829814.119578:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d24f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646867712:192.168.203.143@tcp:8:kworker.0 00000100:00000001:3.0:1787829814.119580:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829814.119582:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829814.119583:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829814.119584:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829814.119585:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829814.119586:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829814.119587:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829814.119588:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829814.119595:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829814.119598:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829814.119598:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829814.119626:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829814.119627:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829814.119629:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829814.119630:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829814.119631:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829814.119633:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829814.119634:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829814.119636:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829814.119638:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829814.119639:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829814.119642:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829814.119643:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829814.119644:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829814.119645:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829814.119646:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829814.119647:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829814.119649:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829814.119650:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829814.119652:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829814.119653:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829814.119655:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829814.119659:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787829814.119660:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829814.119662:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1787829814.119664:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829815.014684:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829815.014689:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829815.014745:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829815.014749:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829815.014753:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829816.038759:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829816.038770:0:815:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1787829816.038773:0:815:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1787829816.038777:0:815:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829816.038783:0:815:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1787829816.038786:0:815:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829816.038789:0:815:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829816.038791:0:815:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829816.038861:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829816.038866:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829816.038870:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829817.062711:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829817.062718:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829817.062766:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829817.062771:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829817.062775:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829818.086685:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829818.086692:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829818.086750:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829818.086754:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829818.086758:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829819.110721:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829819.110729:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829819.110804:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829819.110808:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829819.110812:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787829819.238728:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829819.238738:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1787829819.238741:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787829819.238745:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787829819.238750:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787829819.238757:0:40701:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 4619 00000100:00080000:3.0:1787829819.238768:0:40701:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787829819.238771:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829819.238773:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:3.0:1787829819.238777:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1787829819.238780:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1787829819.238783:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:3.0:1787829819.238786:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:3.0:1787829819.238788:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829819.238790:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:3.0:1787829819.238794:0:40701:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:3.0:1787829819.238798:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829819.238800:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787829819.238804:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829819.238807:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829819.238810:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:3.0:1787829819.238813:0:40701:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829819.238824:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787829819.238826:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829819.238829:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787829819.238831:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787829819.238836:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829819.238847:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829819.238857:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c70170c0 x1874670646867840/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787829819.238868:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787829819.238873:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829819.238893:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787829819.238896:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829819.238900:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1787829819.238902:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787829819.238903:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787829819.238907:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787829819.238911:0:40701:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4619 00000100:00080000:3.0:1787829819.238915:0:40701:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787829819.238918:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829819.238920:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1787829819.238921:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:0.0:1787829819.238926:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000020:00000001:3.0:1787829819.238926:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:0.0:1787829819.238930:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000020:00000001:3.0:1787829819.238931:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:0.0:1787829819.238933:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000020:00000001:3.0:1787829819.238935:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:0.0:1787829819.238936:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829819.238942:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000020:00000001:3.0:1787829819.238943:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 02000000:00000001:0.0:1787829819.238945:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829819.238947:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 02000000:00000001:0.0:1787829819.238949:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829819.238952:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00100000:0.0:1787829819.238955:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c70170c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646867840:192.168.203.143@tcp:250:kworker.0 00000100:00080000:3.0:1787829819.238960:0:40701:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787829819.238962:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787829819.238967:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:3.0:1787829819.238967:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829819.238971:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829819.238973:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787829819.238974:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1787829819.238979:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829819.238981:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829819.238985:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829819.238988:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1787829819.238992:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000008:00100000:3.0:1787829819.238993:0:40701:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00000001:0.0:1787829819.238995:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369547968 : -126234340003648 : ffff8d30c70170c0) 00000008:00000001:3.0:1787829819.239000:0:40701:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829819.239001:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:3.0:1787829819.239005:0:40701:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829819.239011:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787829819.239012:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829819.239014:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787829819.239016:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787829819.239018:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829819.239024:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829819.239030:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c7016d00 x1874670646867968/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787829819.239038:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:3.0:1787829819.239047:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787829819.239050:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829819.239051:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1787829819.239057:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829819.239058:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829819.239060:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829819.239064:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00080000:3.0:1787829819.239066:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1787829819.239069:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829819.239073:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4299293440 00000100:00000001:0.0:1787829819.239080:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 00000100:00080000:3.0:1787829819.239081:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1787829819.239086:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787829819.239088:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787829819.239089:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4299293440 00000100:00000001:3.0:1787829819.239092:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787829819.239096:0:40701:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4619 00000100:00080000:3.0:1787829819.239100:0:40701:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787829819.239106:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829819.239108:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 00000100:00000001:3.0:1787829819.239109:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:3.0:1787829819.239114:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:0.0:1787829819.239116:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000020:00000001:3.0:1787829819.239117:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1787829819.239120:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:0.0:1787829819.239123:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829819.239127:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000020:00000001:3.0:1787829819.239133:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00100000:0.0:1787829819.239135:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829819/real 1787829819] req@ffff8d30c70170c0 x1874670646867840/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829844 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829819.239138:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829819.239142:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:3.0:1787829819.239153:0:40701:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000400:00000001:0.0:1787829819.239156:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000001:3.0:1787829819.239158:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:0.0:1787829819.239160:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 02000000:00000001:3.0:1787829819.239163:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:0.0:1787829819.239166:0:2390:0:(events.c:78:reply_in_callback()) Process entered 02000000:00000001:3.0:1787829819.239168:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787829819.239171:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 02000000:00000001:3.0:1787829819.239173:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829819.239174:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829819.239178:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1787829819.239180:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000008:00100000:3.0:1787829819.239183:0:40701:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00000001:0.0:1787829819.239184:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000008:00000001:3.0:1787829819.239191:0:40701:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787829819.239196:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c70170c0 x1874670646867840/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829844 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829819.239196:0:40701:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829819.239200:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787829819.239204:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829819.239210:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:0.0:1787829819.239213:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:3.0:1787829819.239213:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787829819.239219:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829819.239220:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829819.239224:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829819.239225:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829819.239230:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787829819.239233:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829819.239235:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c70143c0 x1874670646868096/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787829819.239237:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787829819.239240:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787829819.239248:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00000001:3.0:1787829819.239255:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00080000:0.0:1787829819.239257:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829819.239261:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829819.239263:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000001:3.0:1787829819.239267:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787829819.239272:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c70170c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646867840:192.168.203.143@tcp:250:kworker.0 00000100:00080000:3.0:1787829819.239272:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1787829819.239282:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:3.0:1787829819.239283:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1787829819.239287:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:0.0:1787829819.239290:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00080000:3.0:1787829819.239291:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787829819.239294:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 00000100:00000001:3.0:1787829819.239296:0:40701:0:(import.c:485:import_select_connection()) Process entered 02000000:00000001:0.0:1787829819.239298:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 00000100:00080000:3.0:1787829819.239302:0:40701:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4619 02000000:00000001:0.0:1787829819.239303:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787829819.239307:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000100:00080000:3.0:1787829819.239309:0:40701:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000020:00000001:0.0:1787829819.239311:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 00000100:00000001:3.0:1787829819.239314:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 02000000:00000001:0.0:1787829819.239317:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 00000100:00000001:3.0:1787829819.239318:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 02000000:00000001:0.0:1787829819.239322:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000020:00000001:3.0:1787829819.239323:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:0.0:1787829819.239327:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000020:00000001:3.0:1787829819.239328:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:0.0:1787829819.239331:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:3.0:1787829819.239332:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:0.0:1787829819.239336:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:3.0:1787829819.239338:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:3.0:1787829819.239344:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829819.239345:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:0.0:1787829819.239347:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00080000:3.0:1787829819.239351:0:40701:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787829819.239352:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829819.239354:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829819.239357:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:3.0:1787829819.239358:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:0.0:1787829819.239363:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 02000000:00000001:3.0:1787829819.239363:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787829819.239367:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 02000000:00000001:3.0:1787829819.239368:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829819.239373:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787829819.239373:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:3.0:1787829819.239376:0:40701:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00000001:0.0:1787829819.239377:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:3.0:1787829819.239382:0:40701:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829819.239384:0:40701:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829819.239387:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787829819.239388:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829819.239390:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787829819.239391:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787829819.239392:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787829819.239392:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829819.239398:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829819.239413:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829819.239420:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829819.239421:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c70152c0 x1874670646868224/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829819.239429:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c7016d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646867968:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829819.239431:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787829819.239436:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829819.239439:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:3.0:1787829819.239441:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829819.239443:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787829819.239447:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829819.239450:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1787829819.239461:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829819.239469:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787829819.239471:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369547008 : -126234340004608 : ffff8d30c7016d00) 00000100:00000001:0.0:1787829819.239476:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1787829819.239491:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:0.0:1787829819.239496:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1787829819.239508:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787829819.239509:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829819.239511:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787829819.239514:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829819.239519:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4299292182 00000100:00000001:0.0:1787829819.239522:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4299293440 00000100:00000001:0.0:1787829819.239525:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4299292182 00000100:00000001:0.0:1787829819.239527:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4299293440 00000100:00000001:0.0:1787829819.239529:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787829819.239532:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829819.239534:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787829819.239539:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829819/real 1787829819] req@ffff8d30c7016d00 x1874670646867968/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829844 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787829819.239548:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787829819.239549:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787829819.239552:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787829819.239554:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787829819.239555:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829819.239557:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787829819.239558:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787829819.239563:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c7016d00 x1874670646867968/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829844 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787829819.239570:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829819.239572:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787829819.239574:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829819.239576:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787829819.239577:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829819.239579:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787829819.239581:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787829819.239584:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829819.239588:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787829819.239591:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787829819.239594:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c7016d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646867968:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787829819.239598:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1787829819.239620:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787829819.239622:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829819.239625:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787829819.239627:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787829819.239629:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787829819.239632:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c70143c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646868096:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787829819.239636:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787829819.239638:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787829819.239640:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829819.239642:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1787829819.239646:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829819.239651:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787829819.239653:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369536448 : -126234340015168 : ffff8d30c70143c0) 00000100:00000001:0.0:1787829819.239657:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1787829819.239670:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:0.0:1787829819.239675:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1787829819.239678:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787829819.239679:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829819.239681:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787829819.239684:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829819.239688:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4299292183 00000100:00000001:0.0:1787829819.239690:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4299293440 00000100:00000001:0.0:1787829819.239693:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4299292183 00000100:00000001:0.0:1787829819.239695:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4299293440 00000100:00000001:0.0:1787829819.239697:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787829819.239699:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829819.239701:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787829819.239705:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829819/real 1787829819] req@ffff8d30c70143c0 x1874670646868096/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829844 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787829819.239714:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787829819.239715:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787829819.239718:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787829819.239720:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787829819.239721:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829819.239723:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787829819.239725:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787829819.239729:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c70143c0 x1874670646868096/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829844 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787829819.239735:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829819.239737:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787829819.239740:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829819.239741:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787829819.239742:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829819.239744:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787829819.239746:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787829819.239748:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829819.239751:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787829819.239753:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787829819.239755:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c70143c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646868096:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787829819.239759:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829819.239760:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787829819.239762:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787829819.239763:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787829819.239765:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787829819.239767:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787829819.239768:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787829819.239770:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787829819.239771:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787829819.239774:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787829819.239775:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829819.239776:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787829819.239777:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787829819.239779:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787829819.239780:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787829819.239781:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787829819.239782:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787829819.239784:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787829819.239785:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787829819.239787:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787829819.239788:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829819.239790:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829819.239793:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787829819.239795:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829819.239797:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787829819.239799:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787829819.239800:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1787829819.239802:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787829819.239804:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829819.239806:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787829819.239808:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787829819.239810:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787829819.239813:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c70152c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646868224:192.168.203.143@tcp:8:kworker.0 00000100:00000001:0.0:1787829819.239817:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787829819.239819:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787829819.239820:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829819.239822:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1787829819.239825:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829819.239831:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787829819.239833:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369540288 : -126234340011328 : ffff8d30c70152c0) 00000100:00000001:0.0:1787829819.239837:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1787829819.239848:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:0.0:1787829819.239853:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1787829819.239856:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787829819.239858:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829819.239859:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787829819.239862:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829819.239866:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4299292196 00000100:00000001:0.0:1787829819.239869:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4299293440 00000100:00000001:0.0:1787829819.239871:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4299292196 00000100:00000001:0.0:1787829819.239873:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4299293440 00000100:00000001:0.0:1787829819.239875:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787829819.239877:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829819.239879:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787829819.239883:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829819/real 1787829819] req@ffff8d30c70152c0 x1874670646868224/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829844 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787829819.239892:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787829819.239893:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787829819.239896:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787829819.239897:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787829819.239899:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829819.239901:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787829819.239903:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787829819.239906:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c70152c0 x1874670646868224/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829844 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787829819.239914:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829819.239916:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787829819.239918:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829819.239920:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787829819.239921:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829819.239923:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787829819.239926:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787829819.239928:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829819.239932:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787829819.239934:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787829819.239937:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c70152c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646868224:192.168.203.143@tcp:8:kworker.0 00000100:00000001:0.0:1787829819.239941:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829819.239943:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787829819.239944:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787829819.239946:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787829819.239948:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787829819.239949:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787829819.239950:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787829819.239953:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787829819.239954:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787829819.239956:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787829819.239957:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829819.239959:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829819.239962:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787829819.239964:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829819.239966:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787829819.239968:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829820.134680:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829820.134686:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829820.134733:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829820.134736:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829820.134739:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829821.158704:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829821.158709:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829821.158773:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829821.158777:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829821.158781:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829822.182681:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829822.182689:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829822.182736:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829822.182740:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829822.182742:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829823.206698:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829823.206705:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829823.206754:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829823.206757:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829823.206759:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829824.230704:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829824.230709:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829824.230758:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829824.230761:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829824.230763:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787829824.358724:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829824.358733:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1787829824.358736:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787829824.358739:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787829824.358743:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787829824.358750:0:40701:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 4624 00000100:00080000:3.0:1787829824.358760:0:40701:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787829824.358762:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829824.358764:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:3.0:1787829824.358767:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1787829824.358769:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1787829824.358771:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:3.0:1787829824.358773:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:3.0:1787829824.358774:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829824.358775:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:3.0:1787829824.358779:0:40701:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:3.0:1787829824.358780:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829824.358782:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787829824.358785:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829824.358787:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829824.358788:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:3.0:1787829824.358790:0:40701:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829824.358800:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787829824.358801:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829824.358803:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787829824.358806:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787829824.358810:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829824.358819:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829824.358827:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c7015680 x1874670646868352/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787829824.358838:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787829824.358842:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829824.358855:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787829824.358857:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829824.358860:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1787829824.358861:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787829824.358862:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787829824.358864:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787829824.358865:0:40701:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4624 00000100:00080000:3.0:1787829824.358868:0:40701:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787829824.358869:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829824.358871:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:3.0:1787829824.358872:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1787829824.358873:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1787829824.358875:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000020:00000001:3.0:1787829824.358876:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:3.0:1787829824.358878:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829824.358880:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:3.0:1787829824.358882:0:40701:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:3.0:1787829824.358884:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829824.358885:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787829824.358889:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787829824.358890:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:3.0:1787829824.358891:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829824.358895:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1787829824.358897:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000008:00100000:3.0:1787829824.358899:0:40701:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00000001:0.0:1787829824.358901:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000008:00000001:3.0:1787829824.358905:0:40701:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829824.358906:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787829824.358910:0:40701:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829824.358911:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829824.358917:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787829824.358918:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787829824.358919:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787829824.358922:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829824.358922:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829824.358927:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00100000:0.0:1787829824.358930:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c7015680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646868352:192.168.203.143@tcp:250:kworker.0 02000000:00000001:3.0:1787829824.358931:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787829824.358935:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829824.358936:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787829824.358940:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:3.0:1787829824.358942:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829824.358943:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829824.358946:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00100000:3.0:1787829824.358949:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c7014b40 x1874670646868480/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:0.0:1787829824.358950:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829824.358958:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787829824.358960:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829824.358962:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829824.358964:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00080000:3.0:1787829824.358964:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829824.358967:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1787829824.358968:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369541248 : -126234340010368 : ffff8d30c7015680) 00000100:00000001:3.0:1787829824.358971:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:0.0:1787829824.358973:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00080000:3.0:1787829824.358973:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787829824.358977:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787829824.358979:0:40701:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4624 00000100:00080000:3.0:1787829824.358982:0:40701:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787829824.358983:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829824.358984:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:3.0:1787829824.358987:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1787829824.358987:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1787829824.358989:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000020:00000001:3.0:1787829824.358990:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:3.0:1787829824.358992:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829824.358993:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:3.0:1787829824.358995:0:40701:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:3.0:1787829824.358996:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829824.358997:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787829824.358999:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829824.359000:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829824.359002:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:3.0:1787829824.359003:0:40701:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1787829824.359005:0:40701:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829824.359006:0:40701:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829824.359008:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787829824.359009:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829824.359010:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787829824.359011:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787829824.359012:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829824.359014:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829824.359016:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829824.359020:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4299298560 00000100:00100000:3.0:1787829824.359021:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c7014f00 x1874670646868608/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787829824.359029:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787829824.359030:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829824.359032:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829824.359033:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 00000100:00080000:3.0:1787829824.359034:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829824.359038:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1787829824.359039:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787829824.359040:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787829824.359043:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4299298560 00000100:00000001:3.0:1787829824.359043:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787829824.359047:0:40701:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4624 00000100:00000001:0.0:1787829824.359050:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 00000100:00080000:3.0:1787829824.359050:0:40701:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787829824.359054:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829824.359055:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829824.359055:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:3.0:1787829824.359060:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:0.0:1787829824.359061:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829824.359061:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1787829824.359064:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000020:00000001:3.0:1787829824.359066:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:0.0:1787829824.359067:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829824.359068:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829824.359071:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:0.0:1787829824.359073:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:3.0:1787829824.359074:0:40701:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:3.0:1787829824.359079:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829824.359080:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787829824.359081:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829824.359082:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829824.359084:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787829824.359084:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:3.0:1787829824.359087:0:40701:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00000001:0.0:1787829824.359088:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000008:00000001:3.0:1787829824.359090:0:40701:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829824.359092:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1787829824.359094:0:40701:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829824.359096:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787829824.359099:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 02000000:00000001:3.0:1787829824.359100:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:0.0:1787829824.359101:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787829824.359104:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829824.359105:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829824.359106:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787829824.359109:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829824.359109:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787829824.359112:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787829824.359114:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829824.359116:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787829824.359121:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c7014b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646868480:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829824.359125:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829824.359127:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787829824.359131:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000100:00100000:3.0:1787829824.359132:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c70161c0 x1874670646868736/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:0.0:1787829824.359134:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829824.359139:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:3.0:1787829824.359143:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 02000000:00000001:0.0:1787829824.359145:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829824.359145:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829824.359149:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829824.359155:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787829824.359158:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369538368 : -126234340013248 : ffff8d30c7014b40) 00000100:00000001:0.0:1787829824.359164:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:0.0:1787829824.359181:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829824.359186:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4299297302 00000100:00000001:0.0:1787829824.359189:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4299298560 00000100:00000001:0.0:1787829824.359191:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4299297302 00000100:00000001:0.0:1787829824.359193:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4299298560 00000100:00000001:0.0:1787829824.359195:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787829824.359197:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829824.359199:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1787829824.359200:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787829824.359201:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829824.359203:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787829824.359204:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787829824.359206:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787829824.359210:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c7014f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646868608:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787829824.359213:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787829824.359214:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787829824.359216:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829824.359217:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1787829824.359220:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829824.359226:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787829824.359227:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369539328 : -126234340012288 : ffff8d30c7014f00) 00000100:00000001:0.0:1787829824.359231:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:0.0:1787829824.359240:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829824.359243:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4299297302 00000100:00000001:0.0:1787829824.359245:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4299298560 00000100:00000001:0.0:1787829824.359247:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4299297302 00000100:00000001:0.0:1787829824.359249:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4299298560 00000100:00000001:0.0:1787829824.359251:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787829824.359253:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829824.359255:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829824.359257:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829824.359260:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787829824.359262:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:0.0:1787829824.359263:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787829824.359264:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787829824.359265:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1787829824.359266:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787829824.359267:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829824.359269:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787829824.359271:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787829824.359272:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787829824.359276:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c70161c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646868736:192.168.203.143@tcp:8:kworker.0 00000100:00000001:0.0:1787829824.359279:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787829824.359281:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787829824.359282:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829824.359283:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1787829824.359286:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829824.359291:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787829824.359292:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369544128 : -126234340007488 : ffff8d30c70161c0) 00000100:00000001:0.0:1787829824.359295:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:0.0:1787829824.359304:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829824.359308:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4299297315 00000100:00000001:0.0:1787829824.359310:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4299298560 00000100:00000001:0.0:1787829824.359311:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4299297315 00000100:00000001:0.0:1787829824.359313:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4299298560 00000100:00000001:0.0:1787829824.359315:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787829824.359317:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829824.359319:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829824.359320:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829824.359323:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787829824.359324:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:0.0:1787829824.359326:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787829824.359327:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787829824.359328:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829824.359330:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829825.254688:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829825.254692:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829825.254745:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829825.254749:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829825.254753:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829826.278666:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829826.278672:0:815:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1787829826.278674:0:815:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1787829826.278676:0:815:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829826.278679:0:815:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1787829826.278684:0:815:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829826.278686:0:815:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829826.278687:0:815:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829826.278727:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829826.278730:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829826.278732:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829827.302673:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829827.302679:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829827.302725:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829827.302728:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829827.302731:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:1.0:1787829827.430809:0:2378:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.143:988 00000400:00000100:1.0:1787829827.430844:0:2378:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.143@tcp at host 192.168.203.143 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1787829827.430857:0:2378:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.143@tcp 00000100:00000001:1.0:1787829827.430869:0:2378:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787829827.430886:0:2378:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829827.430889:0:2378:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829827.430891:0:2378:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:1.0:1787829827.430900:0:2378:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.143@tcp 00000100:00000001:1.0:1787829827.430904:0:2378:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787829827.430906:0:2378:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829827.430907:0:2378:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829827.430908:0:2378:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:1.0:1787829827.430912:0:2378:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.143@tcp 00000100:00000001:1.0:1787829827.430917:0:2378:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:1.0:1787829827.430920:0:2378:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:1.0:1787829827.430922:0:2378:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829827.430924:0:2378:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:1.0:1787829827.430930:0:2378:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.143@tcp 00000100:00000001:0.0:1787829827.430933:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:1.0:1787829827.430935:0:2378:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1787829827.430939:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:1.0:1787829827.430940:0:2378:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787829827.430943:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00000001:1.0:1787829827.430943:0:2378:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:1.0:1787829827.430947:0:2378:0:(events.c:68:request_out_callback()) Process leaving 00000100:00100000:0.0:1787829827.430956:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829824/real 1787829827] req@ffff8d30c70161c0 x1874670646868736/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829849 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787829827.430969:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787829827.430971:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787829827.430976:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787829827.430979:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787829827.430981:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829827.430985:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787829827.430986:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787829827.430991:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c70161c0 x1874670646868736/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829849 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787829827.430998:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829827.431002:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787829827.431005:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829827.431006:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787829827.431007:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829827.431010:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787829827.431012:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787829827.431016:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829827.431020:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787829827.431023:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787829827.431027:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c70161c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646868736:192.168.203.143@tcp:8:kworker.0 00000100:00000001:0.0:1787829827.431032:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787829827.431037:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829824/real 1787829827] req@ffff8d30c7014b40 x1874670646868480/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829849 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787829827.431053:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787829827.431054:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787829827.431058:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787829827.431060:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787829827.431061:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829827.431063:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787829827.431064:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787829827.431068:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c7014b40 x1874670646868480/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829849 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787829827.431075:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829827.431077:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787829827.431079:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829827.431081:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787829827.431082:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829827.431083:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787829827.431085:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787829827.431088:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829827.431091:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787829827.431093:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787829827.431097:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c7014b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646868480:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787829827.431101:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787829827.431105:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829824/real 1787829827] req@ffff8d30c7014f00 x1874670646868608/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829849 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787829827.431112:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787829827.431113:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787829827.431115:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787829827.431117:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787829827.431118:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829827.431120:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787829827.431122:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787829827.431125:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c7014f00 x1874670646868608/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829849 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787829827.431132:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829827.431134:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787829827.431136:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829827.431137:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787829827.431138:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829827.431140:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787829827.431142:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787829827.431144:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829827.431147:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787829827.431149:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787829827.431152:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c7014f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646868608:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787829827.431157:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787829827.431160:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829824/real 1787829827] req@ffff8d30c7015680 x1874670646868352/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829849 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787829827.431168:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787829827.431169:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787829827.431172:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787829827.431174:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787829827.431175:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829827.431177:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787829827.431178:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787829827.431182:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c7015680 x1874670646868352/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829849 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787829827.431188:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829827.431190:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787829827.431192:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829827.431194:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787829827.431195:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829827.431197:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787829827.431198:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787829827.431200:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829827.431203:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787829827.431205:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787829827.431208:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c7015680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646868352:192.168.203.143@tcp:250:kworker.0 00000100:00000001:0.0:1787829827.431212:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829827.431215:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787829827.431216:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787829827.431219:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787829827.431223:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787829827.431226:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787829827.431228:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787829827.431231:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787829827.431232:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787829827.431235:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787829827.431236:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829827.431238:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787829827.431239:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787829827.431240:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787829827.431242:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787829827.431243:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787829827.431244:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787829827.431246:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787829827.431247:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787829827.431249:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787829827.431250:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829827.431251:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787829827.431252:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787829827.431254:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787829827.431256:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787829827.431257:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787829827.431258:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787829827.431259:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787829827.431260:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787829827.431262:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787829827.431264:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829827.431266:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787829827.431267:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787829827.431268:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787829827.431269:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787829827.431270:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787829827.431272:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787829827.431273:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787829827.431274:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787829827.431276:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787829827.431277:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829827.431279:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829827.431287:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787829827.431289:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829827.431291:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787829827.431292:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829828.326684:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829828.326689:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829828.326725:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829828.326729:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829828.326731:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829829.350757:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829829.350766:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829829.350833:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829829.350838:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829829.350842:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787829829.478763:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829829.478774:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1787829829.478776:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787829829.478780:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787829829.478784:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787829829.478792:0:40701:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 4629 00000100:00080000:0.0:1787829829.478802:0:40701:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787829829.478806:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829829.478809:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787829829.478813:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787829829.478818:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787829829.478822:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:0.0:1787829829.478825:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:0.0:1787829829.478827:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829829.478829:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:0.0:1787829829.478834:0:40701:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787829829.478844:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829829.478847:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787829829.478851:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787829829.478854:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829829.478858:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1787829829.478862:0:40701:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829829.478874:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787829829.478877:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829829.478880:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787829829.478882:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787829829.478886:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829829.478899:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787829829.478909:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d24f00 x1874670646868864/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787829829.478919:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787829829.478924:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829829.478949:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787829829.478952:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829829.478956:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1787829829.478957:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787829829.478960:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787829829.478963:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787829829.478967:0:40701:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4629 00000100:00080000:0.0:1787829829.478971:0:40701:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787829829.478973:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829829.478975:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787829829.478977:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787829829.478978:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787829829.478980:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000020:00000001:0.0:1787829829.478982:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:0.0:1787829829.478984:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829829.478985:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:0.0:1787829829.478988:0:40701:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787829829.478991:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829829.478993:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787829829.478998:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787829829.479000:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829829.479002:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1787829829.479012:0:40701:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1787829829.479016:0:40701:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829829.479018:0:40701:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829829.479023:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787829829.479025:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829829.479027:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787829829.479028:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787829829.479031:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829829.479038:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787829829.479044:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d24000 x1874670646868992/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787829829.479051:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787829829.479053:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829829.479056:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787829829.479059:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829829.479063:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1787829829.479064:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787829829.479067:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787829829.479070:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787829829.479073:0:40701:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4629 00000100:00080000:0.0:1787829829.479077:0:40701:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787829829.479079:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829829.479080:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787829829.479082:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787829829.479083:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787829829.479085:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000020:00000001:0.0:1787829829.479086:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:0.0:1787829829.479087:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829829.479089:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:0.0:1787829829.479091:0:40701:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787829829.479093:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829829.479095:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787829829.479097:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787829829.479101:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829829.479103:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1787829829.479107:0:40701:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1787829829.479111:0:40701:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829829.479114:0:40701:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829829.479121:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787829829.479124:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829829.479127:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787829829.479129:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787829829.479132:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829829.479138:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787829829.479141:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829829.479147:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00100000:0.0:1787829829.479149:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d270c0 x1874670646869120/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787829829.479152:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1787829829.479156:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787829829.479161:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787829829.479161:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829829.479167:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829829.479172:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:0.0:1787829829.479176:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829829.479178:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00080000:0.0:1787829829.479181:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 02000000:00000001:2.0:1787829829.479184:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787829829.479188:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1787829829.479192:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787829829.479195:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1787829829.479197:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d24f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646868864:192.168.203.143@tcp:250:kworker.0 00000100:00000001:0.0:1787829829.479199:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:2.0:1787829829.479208:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00080000:0.0:1787829829.479210:0:40701:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4629 00000100:00080000:0.0:1787829829.479216:0:40701:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 02000000:00000001:2.0:1787829829.479216:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1787829829.479221:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829829.479222:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829829.479227:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 02000000:00000001:2.0:1787829829.479230:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000020:00000001:0.0:1787829829.479232:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787829829.479237:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787829829.479239:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 02000000:00000001:2.0:1787829829.479239:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787829829.479247:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:0.0:1787829829.479249:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829829.479250:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:0.0:1787829829.479255:0:40701:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787829829.479258:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829829.479259:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787829829.479262:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787829829.479264:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829829.479265:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787829829.479268:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:2.0:1787829829.479270:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601340160 : -126234108211456 : ffff8d30d4d24f00) 00000008:00100000:0.0:1787829829.479274:0:40701:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00000001:2.0:1787829829.479278:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000008:00000001:0.0:1787829829.479282:0:40701:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829829.479286:0:40701:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829829.479294:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787829829.479295:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829829.479297:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787829829.479298:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787829829.479300:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829829.479306:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787829829.479311:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d25a40 x1874670646869248/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787829829.479317:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787829829.479319:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829829.479321:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:2.0:1787829829.479356:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:2.0:1787829829.479375:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1787829829.479381:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787829829.479384:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829829.479387:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1787829829.479394:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829829.479400:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4299303680 00000100:00000001:2.0:1787829829.479413:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 00000100:00000001:2.0:1787829829.479420:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4299303680 00000100:00000001:2.0:1787829829.479425:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 00000100:00000001:2.0:1787829829.479430:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787829829.479433:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829829.479436:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1787829829.479443:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829829/real 1787829829] req@ffff8d30d4d24f00 x1874670646868864/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829854 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1787829829.479456:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787829829.479457:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1787829829.479466:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1787829829.479470:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787829829.479473:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829829.479477:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787829829.479480:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787829829.479487:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d24f00 x1874670646868864/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829854 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787829829.479496:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829829.479500:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787829829.479503:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829829.479505:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787829829.479506:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829829.479509:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787829829.479513:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787829829.479517:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829829.479522:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:2.0:1787829829.479525:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1787829829.479529:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d24f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646868864:192.168.203.143@tcp:250:kworker.0 00000100:00000001:2.0:1787829829.479535:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1787829829.479536:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787829829.479538:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829829.479541:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787829829.479543:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787829829.479546:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787829829.479549:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d24000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646868992:192.168.203.143@tcp:38:kworker.0 00000100:00000001:2.0:1787829829.479553:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787829829.479555:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1787829829.479557:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829829.479559:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1787829829.479563:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829829.479569:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787829829.479571:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601336320 : -126234108215296 : ffff8d30d4d24000) 00000100:00000001:2.0:1787829829.479576:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1787829829.479591:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:2.0:1787829829.479649:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1787829829.479652:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787829829.479653:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829829.479655:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1787829829.479658:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829829.479664:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4299302423 00000100:00000001:2.0:1787829829.479667:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4299303680 00000100:00000001:2.0:1787829829.479669:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4299302423 00000100:00000001:2.0:1787829829.479671:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4299303680 00000100:00000001:2.0:1787829829.479673:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787829829.479675:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829829.479677:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1787829829.479682:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829829/real 1787829829] req@ffff8d30d4d24000 x1874670646868992/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829854 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1787829829.479690:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787829829.479691:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1787829829.479694:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1787829829.479696:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787829829.479697:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829829.479699:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787829829.479700:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787829829.479704:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d24000 x1874670646868992/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829854 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787829829.479716:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829829.479718:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787829829.479720:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829829.479722:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787829829.479723:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829829.479724:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787829829.479726:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787829829.479736:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829829.479739:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:2.0:1787829829.479741:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1787829829.479744:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d24000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646868992:192.168.203.143@tcp:38:kworker.0 00000100:00000001:2.0:1787829829.479748:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829829.479750:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787829829.479753:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787829829.479755:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1787829829.479759:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787829829.479762:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:2.0:1787829829.479765:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:2.0:1787829829.479770:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787829829.479772:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1787829829.479777:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787829829.479778:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829829.479780:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787829829.479781:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787829829.479782:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1787829829.479783:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787829829.479785:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:2.0:1787829829.479786:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:2.0:1787829829.479787:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787829829.479789:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1787829829.479790:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787829829.479791:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829829.479793:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829829.479802:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1787829829.479805:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829829.479808:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787829829.479809:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1787829829.479811:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1787829829.479812:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787829829.479813:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829829.479816:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787829829.479817:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787829829.479819:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787829829.479823:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d270c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646869120:192.168.203.143@tcp:38:kworker.0 00000100:00000001:2.0:1787829829.479826:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787829829.479828:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1787829829.479830:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829829.479832:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1787829829.479834:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829829.479839:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787829829.479840:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601348800 : -126234108202816 : ffff8d30d4d270c0) 00000100:00000001:2.0:1787829829.479844:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1787829829.479855:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:2.0:1787829829.479860:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1787829829.479862:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787829829.479864:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829829.479866:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1787829829.479868:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829829.479872:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4299302423 00000100:00000001:2.0:1787829829.479875:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4299303680 00000100:00000001:2.0:1787829829.479877:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4299302423 00000100:00000001:2.0:1787829829.479879:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4299303680 00000100:00000001:2.0:1787829829.479881:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787829829.479883:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829829.479885:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1787829829.479889:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829829/real 1787829829] req@ffff8d30d4d270c0 x1874670646869120/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829854 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1787829829.479902:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787829829.479903:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1787829829.479906:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1787829829.479907:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787829829.479909:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829829.479910:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787829829.479912:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787829829.479916:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d270c0 x1874670646869120/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829854 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787829829.479923:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829829.479925:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787829829.479927:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829829.479929:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787829829.479930:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829829.479932:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787829829.479933:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787829829.479936:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829829.479939:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:2.0:1787829829.479942:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1787829829.479945:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d270c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646869120:192.168.203.143@tcp:38:kworker.0 00000100:00000001:2.0:1787829829.479949:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1787829829.479951:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787829829.479952:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829829.479955:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787829829.479957:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787829829.479959:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787829829.479962:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d25a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646869248:192.168.203.143@tcp:8:kworker.0 00000100:00000001:2.0:1787829829.479982:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787829829.479985:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1787829829.479987:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829829.479989:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1787829829.479991:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829829.479996:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787829829.479997:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601343040 : -126234108208576 : ffff8d30d4d25a40) 00000100:00000001:2.0:1787829829.480001:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1787829829.480012:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:2.0:1787829829.480017:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1787829829.480019:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787829829.480020:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829829.480022:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1787829829.480024:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829829.480028:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4299302436 00000100:00000001:2.0:1787829829.480030:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4299303680 00000100:00000001:2.0:1787829829.480033:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4299302436 00000100:00000001:2.0:1787829829.480035:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4299303680 00000100:00000001:2.0:1787829829.480037:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787829829.480039:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829829.480041:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1787829829.480045:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829829/real 1787829829] req@ffff8d30d4d25a40 x1874670646869248/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829854 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1787829829.480054:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787829829.480055:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1787829829.480058:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1787829829.480060:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787829829.480062:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829829.480063:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787829829.480065:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787829829.480068:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d25a40 x1874670646869248/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829854 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787829829.480075:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829829.480077:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787829829.480080:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829829.480082:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787829829.480083:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829829.480084:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787829829.480086:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787829829.480089:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829829.480092:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:2.0:1787829829.480094:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1787829829.480096:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d25a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646869248:192.168.203.143@tcp:8:kworker.0 00000100:00000001:2.0:1787829829.480100:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829829.480102:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787829829.480104:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787829829.480105:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1787829829.480107:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787829829.480109:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:2.0:1787829829.480110:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:2.0:1787829829.480112:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787829829.480114:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1787829829.480118:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787829829.480120:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829829.480122:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787829829.480123:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787829829.480124:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1787829829.480126:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787829829.480126:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:2.0:1787829829.480128:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:2.0:1787829829.480130:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787829829.480131:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1787829829.480133:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787829829.480134:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829829.480135:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829829.480138:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1787829829.480140:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829829.480142:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787829829.480144:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829830.374743:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829830.374752:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829830.374821:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829830.374824:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829830.374828:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829831.398751:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829831.398760:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829831.398838:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829831.398843:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829831.398847:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829832.422739:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829832.422745:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829832.422815:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829832.422818:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829832.422821:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829833.446719:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829833.446724:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829833.446772:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829833.446776:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829833.446779:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829834.470697:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829834.470701:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829834.470759:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829834.470764:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829834.470773:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787829834.598728:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829834.598740:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1787829834.598742:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787829834.598746:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787829834.598750:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787829834.598757:0:40798:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 4635 00000100:00080000:0.0:1787829834.598768:0:40798:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787829834.598771:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829834.598773:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787829834.598776:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787829834.598779:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787829834.598781:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:0.0:1787829834.598783:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:0.0:1787829834.598784:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829834.598785:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:0.0:1787829834.598789:0:40798:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787829834.598790:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829834.598792:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787829834.598795:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787829834.598797:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829834.598799:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1787829834.598801:0:40798:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829834.598812:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787829834.598813:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829834.598816:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787829834.598819:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787829834.598823:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829834.598833:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787829834.598841:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d252c0 x1874670646869376/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787829834.598850:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787829834.598853:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829834.598871:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787829834.598873:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829834.598875:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1787829834.598876:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787829834.598877:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787829834.598879:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787829834.598880:0:40798:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4635 00000100:00080000:0.0:1787829834.598883:0:40798:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787829834.598884:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829834.598885:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787829834.598887:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787829834.598888:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787829834.598889:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000020:00000001:0.0:1787829834.598891:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:0.0:1787829834.598892:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829834.598893:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:0.0:1787829834.598894:0:40798:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787829834.598895:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829834.598896:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787829834.598901:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787829834.598902:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829834.598904:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1787829834.598908:0:40798:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1787829834.598910:0:40798:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829834.598911:0:40798:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829834.598914:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787829834.598914:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829834.598915:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787829834.598916:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787829834.598917:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829834.598920:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787829834.598925:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829834.598925:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1787829834.598928:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00100000:0.0:1787829834.598931:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d25680 x1874670646869504/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787829834.598932:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787829834.598936:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829834.598942:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 02000000:00000001:2.0:1787829834.598942:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:0.0:1787829834.598945:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829834.598948:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829834.598949:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00080000:0.0:1787829834.598950:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829834.598955:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 02000000:00000001:2.0:1787829834.598955:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829834.598957:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787829834.598960:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787829834.598962:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787829834.598965:0:40798:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4635 00000100:00080000:0.0:1787829834.598967:0:40798:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787829834.598968:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829834.598973:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787829834.598975:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000100:00100000:2.0:1787829834.598976:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d252c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646869376:192.168.203.143@tcp:250:kworker.0 00000020:00000001:0.0:1787829834.598977:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787829834.598981:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000020:00000001:0.0:1787829834.598986:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:2.0:1787829834.598986:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:0.0:1787829834.598990:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829834.598992:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 02000000:00000001:2.0:1787829834.598993:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000100:00080000:0.0:1787829834.598997:0:40798:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 02000000:00000001:2.0:1787829834.598998:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829834.599002:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829834.599004:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1787829834.599006:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787829834.599010:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787829834.599011:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829834.599014:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829834.599017:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1787829834.599019:0:40798:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1787829834.599022:0:40798:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829834.599024:0:40798:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829834.599028:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787829834.599030:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829834.599032:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:2.0:1787829834.599032:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 02000000:00000001:0.0:1787829834.599036:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:2.0:1787829834.599036:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601341120 : -126234108210496 : ffff8d30d4d252c0) 02000000:00000001:0.0:1787829834.599040:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829834.599043:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:0.0:1787829834.599047:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787829834.599054:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d27c00 x1874670646869632/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787829834.599059:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787829834.599061:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829834.599064:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787829834.599065:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829834.599068:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1787829834.599069:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787829834.599070:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787829834.599072:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787829834.599074:0:40798:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4635 00000100:00080000:0.0:1787829834.599077:0:40798:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787829834.599078:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829834.599079:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787829834.599088:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787829834.599090:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000800:00000100:2.0:1787829834.599090:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000020:00000001:0.0:1787829834.599093:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000020:00000001:0.0:1787829834.599094:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:0.0:1787829834.599096:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829834.599097:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:0.0:1787829834.599100:0:40798:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787829834.599102:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829834.599103:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787829834.599105:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787829834.599107:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829834.599108:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1787829834.599110:0:40798:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1787829834.599112:0:40798:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829834.599113:0:40798:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829834.599116:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:2.0:1787829834.599117:0:2390:0:(events.c:34:request_out_callback()) Process entered 02000000:00000001:0.0:1787829834.599118:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829834.599121:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787829834.599122:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787829834.599124:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829834.599125:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787829834.599128:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829834.599129:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787829834.599134:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d26940 x1874670646869760/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787829834.599134:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1787829834.599142:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829834.599144:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787829834.599147:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829834.599150:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4299308800 00000100:00000001:0.0:1787829834.599151:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829834.599165:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 00000100:00000001:2.0:1787829834.599172:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4299308800 00000100:00000001:2.0:1787829834.599177:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 00000100:00000001:2.0:1787829834.599181:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787829834.599185:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829834.599188:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1787829834.599193:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829834/real 1787829834] req@ffff8d30d4d252c0 x1874670646869376/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829859 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1787829834.599204:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787829834.599206:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1787829834.599210:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1787829834.599213:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787829834.599214:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829834.599217:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787829834.599219:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787829834.599224:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d252c0 x1874670646869376/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829859 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787829834.599232:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829834.599236:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787829834.599239:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829834.599240:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787829834.599242:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829834.599245:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787829834.599247:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787829834.599250:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829834.599255:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:2.0:1787829834.599258:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1787829834.599262:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d252c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646869376:192.168.203.143@tcp:250:kworker.0 00000100:00000001:2.0:1787829834.599266:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829834.599269:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787829834.599270:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787829834.599274:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1787829834.599278:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787829834.599280:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:2.0:1787829834.599282:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:2.0:1787829834.599287:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787829834.599289:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1787829834.599308:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787829834.599310:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829834.599313:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829834.599321:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1787829834.599323:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829834.599325:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787829834.599327:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1787829834.599328:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1787829834.599330:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787829834.599331:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829834.599333:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787829834.599335:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787829834.599337:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787829834.599340:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d25680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646869504:192.168.203.143@tcp:38:kworker.0 00000100:00000001:2.0:1787829834.599343:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787829834.599345:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1787829834.599347:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829834.599348:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1787829834.599351:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829834.599357:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787829834.599358:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601342080 : -126234108209536 : ffff8d30d4d25680) 00000100:00000001:2.0:1787829834.599363:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1787829834.599376:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:2.0:1787829834.599381:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1787829834.599383:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787829834.599385:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829834.599386:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1787829834.599389:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829834.599393:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4299307542 00000100:00000001:2.0:1787829834.599396:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4299308800 00000100:00000001:2.0:1787829834.599399:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4299307542 00000100:00000001:2.0:1787829834.599401:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4299308800 00000100:00000001:2.0:1787829834.599403:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787829834.599405:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829834.599407:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1787829834.599411:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829834/real 1787829834] req@ffff8d30d4d25680 x1874670646869504/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829859 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1787829834.599420:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787829834.599421:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1787829834.599423:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1787829834.599425:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787829834.599426:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829834.599428:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787829834.599429:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787829834.599433:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d25680 x1874670646869504/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829859 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787829834.599441:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829834.599443:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787829834.599445:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829834.599447:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787829834.599448:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829834.599449:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787829834.599451:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787829834.599453:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829834.599457:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:2.0:1787829834.599459:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1787829834.599462:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d25680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646869504:192.168.203.143@tcp:38:kworker.0 00000100:00000001:2.0:1787829834.599465:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1787829834.599467:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787829834.599468:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829834.599470:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787829834.599472:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787829834.599474:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787829834.599476:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d27c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646869632:192.168.203.143@tcp:38:kworker.0 00000100:00000001:2.0:1787829834.599483:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787829834.599484:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1787829834.599486:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829834.599487:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1787829834.599490:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829834.599494:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787829834.599496:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601351680 : -126234108199936 : ffff8d30d4d27c00) 00000100:00000001:2.0:1787829834.599499:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1787829834.599510:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:2.0:1787829834.599515:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1787829834.599517:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787829834.599518:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829834.599520:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1787829834.599522:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829834.599525:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4299307542 00000100:00000001:2.0:1787829834.599528:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4299308800 00000100:00000001:2.0:1787829834.599530:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4299307542 00000100:00000001:2.0:1787829834.599532:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4299308800 00000100:00000001:2.0:1787829834.599534:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787829834.599536:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829834.599537:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1787829834.599542:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829834/real 1787829834] req@ffff8d30d4d27c00 x1874670646869632/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829859 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1787829834.599549:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787829834.599550:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1787829834.599553:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1787829834.599554:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787829834.599556:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829834.599557:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787829834.599559:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787829834.599562:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d27c00 x1874670646869632/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829859 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787829834.599575:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829834.599577:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787829834.599580:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829834.599581:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787829834.599582:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829834.599584:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787829834.599586:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787829834.599588:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829834.599591:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:2.0:1787829834.599593:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1787829834.599596:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d27c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646869632:192.168.203.143@tcp:38:kworker.0 00000100:00000001:2.0:1787829834.599628:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1787829834.599630:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787829834.599631:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829834.599634:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787829834.599636:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787829834.599638:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787829834.599641:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d26940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646869760:192.168.203.143@tcp:8:kworker.0 00000100:00000001:2.0:1787829834.599644:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787829834.599646:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1787829834.599647:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829834.599649:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1787829834.599653:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829834.599660:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787829834.599662:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601346880 : -126234108204736 : ffff8d30d4d26940) 00000100:00000001:2.0:1787829834.599665:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1787829834.599677:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:2.0:1787829834.599681:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1787829834.599684:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787829834.599685:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829834.599688:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1787829834.599690:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829834.599693:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4299307556 00000100:00000001:2.0:1787829834.599696:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4299308800 00000100:00000001:2.0:1787829834.599698:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4299307556 00000100:00000001:2.0:1787829834.599700:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4299308800 00000100:00000001:2.0:1787829834.599702:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787829834.599704:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829834.599705:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1787829834.599709:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829834/real 1787829834] req@ffff8d30d4d26940 x1874670646869760/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829859 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1787829834.599717:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787829834.599718:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1787829834.599721:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1787829834.599722:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787829834.599724:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829834.599725:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787829834.599727:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787829834.599731:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d26940 x1874670646869760/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829859 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787829834.599738:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829834.599740:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787829834.599742:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829834.599744:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787829834.599745:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829834.599747:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787829834.599749:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787829834.599751:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829834.599755:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:2.0:1787829834.599757:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1787829834.599760:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d26940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646869760:192.168.203.143@tcp:8:kworker.0 00000100:00000001:2.0:1787829834.599765:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829834.599767:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787829834.599768:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787829834.599769:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1787829834.599772:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787829834.599773:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:2.0:1787829834.599775:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:2.0:1787829834.599777:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787829834.599778:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1787829834.599781:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787829834.599782:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829834.599784:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787829834.599785:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787829834.599786:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1787829834.599787:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787829834.599789:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:2.0:1787829834.599790:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:2.0:1787829834.599792:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787829834.599793:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1787829834.599796:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787829834.599797:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829834.599799:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787829834.599800:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787829834.599801:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1787829834.599803:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787829834.599804:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:2.0:1787829834.599805:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:2.0:1787829834.599806:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787829834.599807:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1787829834.599810:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787829834.599811:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829834.599821:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829834.599825:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1787829834.599827:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829834.599829:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787829834.599831:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829835.494743:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829835.494753:0:815:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1787829835.494756:0:815:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1787829835.494761:0:815:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829835.494767:0:815:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1787829835.494769:0:815:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829835.494771:0:815:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829835.494774:0:815:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829835.494838:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829835.494842:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829835.494846:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829836.518736:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829836.518743:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829836.518823:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829836.518827:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829836.518830:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829837.542738:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829837.542745:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829837.542836:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829837.542839:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829837.542842:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829838.566751:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829838.566757:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829838.566835:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829838.566840:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829838.566845:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829839.590702:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829839.590709:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829839.590763:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829839.590767:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829839.590770:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787829839.718752:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829839.718765:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1787829839.718768:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787829839.718772:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787829839.718786:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787829839.718795:0:40798:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 4640 00000100:00080000:0.0:1787829839.718808:0:40798:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787829839.718811:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829839.718814:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787829839.718820:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787829839.718823:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787829839.718826:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:0.0:1787829839.718830:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:0.0:1787829839.718832:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829839.718833:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:0.0:1787829839.718839:0:40798:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787829839.718842:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829839.718845:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787829839.718850:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787829839.718853:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829839.718856:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1787829839.718858:0:40798:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829839.718873:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787829839.718875:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829839.718879:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787829839.718882:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787829839.718887:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829839.718902:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787829839.718914:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d26940 x1874670646869888/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787829839.718922:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787829839.718925:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829839.718937:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787829839.718939:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829839.718941:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1787829839.718942:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787829839.718944:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787829839.718946:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787829839.718947:0:40798:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4640 00000100:00080000:0.0:1787829839.718950:0:40798:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787829839.718951:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829839.718952:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787829839.718954:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787829839.718954:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787829839.718956:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000020:00000001:0.0:1787829839.718957:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:0.0:1787829839.718959:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829839.718960:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:0.0:1787829839.718962:0:40798:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787829839.718963:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829839.718964:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787829839.718969:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787829839.718970:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829839.718971:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1787829839.718975:0:40798:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1787829839.718977:0:40798:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829839.718979:0:40798:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829839.718982:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787829839.718984:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829839.718985:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:2.0:1787829839.718985:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:0.0:1787829839.718987:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787829839.718990:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829839.718993:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787829839.718995:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829839.718997:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00100000:0.0:1787829839.719000:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d27c00 x1874670646870016/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787829839.719002:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787829839.719005:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829839.719007:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787829839.719010:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829839.719011:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:0.0:1787829839.719012:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787829839.719015:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 02000000:00000001:2.0:1787829839.719016:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00080000:0.0:1787829839.719019:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1787829839.719022:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 02000000:00000001:2.0:1787829839.719022:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787829839.719023:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787829839.719027:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787829839.719030:0:40798:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4640 00000100:00100000:2.0:1787829839.719031:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d26940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646869888:192.168.203.143@tcp:250:kworker.0 00000100:00080000:0.0:1787829839.719032:0:40798:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787829839.719035:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829839.719037:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787829839.719039:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787829839.719041:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:2.0:1787829839.719041:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000020:00000001:0.0:1787829839.719042:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000020:00000001:0.0:1787829839.719045:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:0.0:1787829839.719046:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829839.719046:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:0.0:1787829839.719048:0:40798:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787829839.719049:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829839.719050:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787829839.719052:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787829839.719053:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829839.719054:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:0.0:1787829839.719055:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1787829839.719057:0:40798:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 02000000:00000001:2.0:1787829839.719058:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:0.0:1787829839.719059:0:40798:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829839.719062:0:40798:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829839.719063:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1787829839.719064:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787829839.719066:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829839.719067:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787829839.719068:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787829839.719069:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829839.719070:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829839.719072:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787829839.719076:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d25680 x1874670646870144/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787829839.719080:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787829839.719081:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829839.719082:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787829839.719084:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829839.719085:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1787829839.719086:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787829839.719087:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787829839.719088:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:2.0:1787829839.719089:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00080000:0.0:1787829839.719090:0:40798:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4640 00000100:00000001:2.0:1787829839.719093:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601346880 : -126234108204736 : ffff8d30d4d26940) 00000100:00080000:0.0:1787829839.719094:0:40798:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787829839.719099:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829839.719099:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787829839.719101:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:2.0:1787829839.719101:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000020:00000001:0.0:1787829839.719104:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787829839.719105:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000020:00000001:0.0:1787829839.719107:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:0.0:1787829839.719108:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829839.719109:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:0.0:1787829839.719111:0:40798:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787829839.719113:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829839.719114:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787829839.719116:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787829839.719117:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829839.719118:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1787829839.719120:0:40798:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1787829839.719121:0:40798:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829839.719122:0:40798:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829839.719125:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787829839.719126:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829839.719127:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787829839.719127:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787829839.719128:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829839.719131:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787829839.719134:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d252c0 x1874670646870272/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787829839.719138:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787829839.719139:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829839.719140:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:2.0:1787829839.719147:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:2.0:1787829839.719163:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1787829839.719167:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787829839.719170:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829839.719171:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1787829839.719175:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829839.719179:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4299313920 00000100:00000001:2.0:1787829839.719188:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 00000100:00000001:2.0:1787829839.719198:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4299313920 00000100:00000001:2.0:1787829839.719201:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 00000100:00000001:2.0:1787829839.719204:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787829839.719206:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829839.719208:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1787829839.719213:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829839/real 1787829839] req@ffff8d30d4d26940 x1874670646869888/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829864 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1787829839.719221:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787829839.719222:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1787829839.719225:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1787829839.719227:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787829839.719229:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829839.719231:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787829839.719232:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787829839.719235:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d26940 x1874670646869888/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829864 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787829839.719240:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829839.719248:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787829839.719250:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829839.719251:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787829839.719252:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829839.719254:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787829839.719256:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787829839.719268:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829839.719271:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:2.0:1787829839.719273:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1787829839.719276:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d26940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646869888:192.168.203.143@tcp:250:kworker.0 00000100:00000001:2.0:1787829839.719281:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829839.719282:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787829839.719284:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787829839.719286:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1787829839.719289:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787829839.719290:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:2.0:1787829839.719292:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:2.0:1787829839.719297:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787829839.719299:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1787829839.719303:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787829839.719305:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829839.719307:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829839.719316:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1787829839.719318:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829839.719320:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787829839.719322:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1787829839.719323:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1787829839.719324:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787829839.719325:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829839.719327:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787829839.719329:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787829839.719331:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787829839.719334:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d27c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646870016:192.168.203.143@tcp:38:kworker.0 00000100:00000001:2.0:1787829839.719338:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787829839.719339:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1787829839.719341:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829839.719343:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1787829839.719345:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829839.719351:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787829839.719352:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601351680 : -126234108199936 : ffff8d30d4d27c00) 00000100:00000001:2.0:1787829839.719356:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1787829839.719367:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:2.0:1787829839.719373:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1787829839.719375:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787829839.719376:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829839.719378:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1787829839.719380:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829839.719385:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4299312662 00000100:00000001:2.0:1787829839.719388:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4299313920 00000100:00000001:2.0:1787829839.719390:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4299312662 00000100:00000001:2.0:1787829839.719392:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4299313920 00000100:00000001:2.0:1787829839.719394:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787829839.719396:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829839.719398:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1787829839.719401:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829839/real 1787829839] req@ffff8d30d4d27c00 x1874670646870016/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829864 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1787829839.719409:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787829839.719410:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1787829839.719412:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1787829839.719414:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787829839.719415:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829839.719416:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787829839.719418:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787829839.719421:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d27c00 x1874670646870016/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829864 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787829839.719427:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829839.719429:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787829839.719431:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829839.719433:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787829839.719434:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829839.719436:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787829839.719437:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787829839.719439:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829839.719442:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:2.0:1787829839.719444:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1787829839.719447:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d27c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646870016:192.168.203.143@tcp:38:kworker.0 00000100:00000001:2.0:1787829839.719450:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1787829839.719451:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787829839.719452:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829839.719454:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787829839.719456:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787829839.719458:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787829839.719461:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d25680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646870144:192.168.203.143@tcp:38:kworker.0 00000100:00000001:2.0:1787829839.719465:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787829839.719466:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1787829839.719468:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829839.719470:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1787829839.719472:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829839.719477:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787829839.719478:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601342080 : -126234108209536 : ffff8d30d4d25680) 00000100:00000001:2.0:1787829839.719481:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1787829839.719492:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:2.0:1787829839.719496:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1787829839.719498:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787829839.719500:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829839.719501:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1787829839.719503:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829839.719507:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4299312662 00000100:00000001:2.0:1787829839.719509:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4299313920 00000100:00000001:2.0:1787829839.719511:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4299312662 00000100:00000001:2.0:1787829839.719513:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4299313920 00000100:00000001:2.0:1787829839.719514:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787829839.719516:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829839.719517:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1787829839.719521:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829839/real 1787829839] req@ffff8d30d4d25680 x1874670646870144/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829864 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1787829839.719528:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787829839.719530:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1787829839.719532:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1787829839.719533:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787829839.719535:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829839.719536:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787829839.719537:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787829839.719541:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d25680 x1874670646870144/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829864 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787829839.719547:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829839.719548:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787829839.719550:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829839.719552:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787829839.719553:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829839.719554:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787829839.719556:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787829839.719558:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829839.719561:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:2.0:1787829839.719563:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1787829839.719567:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d25680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646870144:192.168.203.143@tcp:38:kworker.0 00000100:00000001:2.0:1787829839.719570:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1787829839.719572:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787829839.719573:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829839.719575:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787829839.719577:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787829839.719579:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787829839.719581:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d252c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646870272:192.168.203.143@tcp:8:kworker.0 00000100:00000001:2.0:1787829839.719585:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787829839.719586:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1787829839.719588:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829839.719589:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1787829839.719592:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829839.719597:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787829839.719599:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601341120 : -126234108210496 : ffff8d30d4d252c0) 00000100:00000001:2.0:1787829839.719627:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1787829839.719638:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:2.0:1787829839.719642:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1787829839.719645:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787829839.719646:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829839.719648:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1787829839.719650:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829839.719653:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4299312676 00000100:00000001:2.0:1787829839.719656:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4299313920 00000100:00000001:2.0:1787829839.719658:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4299312676 00000100:00000001:2.0:1787829839.719660:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4299313920 00000100:00000001:2.0:1787829839.719661:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787829839.719663:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829839.719665:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1787829839.719668:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829839/real 1787829839] req@ffff8d30d4d252c0 x1874670646870272/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829864 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1787829839.719674:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787829839.719675:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1787829839.719676:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1787829839.719677:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787829839.719678:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829839.719679:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787829839.719680:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787829839.719682:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d252c0 x1874670646870272/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829864 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787829839.719686:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829839.719687:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787829839.719689:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829839.719690:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787829839.719690:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829839.719691:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787829839.719692:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787829839.719694:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829839.719695:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:2.0:1787829839.719696:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1787829839.719699:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d252c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646870272:192.168.203.143@tcp:8:kworker.0 00000100:00000001:2.0:1787829839.719701:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829839.719702:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787829839.719703:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787829839.719704:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1787829839.719705:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787829839.719706:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:2.0:1787829839.719707:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:2.0:1787829839.719708:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787829839.719709:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1787829839.719712:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787829839.719712:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829839.719713:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787829839.719714:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787829839.719715:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1787829839.719716:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787829839.719717:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:2.0:1787829839.719718:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:2.0:1787829839.719718:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787829839.719719:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1787829839.719721:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787829839.719722:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829839.719722:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787829839.719723:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787829839.719724:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1787829839.719724:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787829839.719725:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:2.0:1787829839.719726:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:2.0:1787829839.719726:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787829839.719727:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1787829839.719736:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787829839.719739:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829839.719740:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829839.719742:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1787829839.719743:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829839.719744:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787829839.719745:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829840.614738:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829840.614745:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829840.614815:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829840.614821:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829840.614824:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829841.638713:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829841.638723:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829841.638792:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829841.638796:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829841.638800:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829842.662752:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829842.662761:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829842.662795:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829842.662799:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829842.662802:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829843.686672:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829843.686678:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829843.686765:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829843.686772:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829843.686779:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829844.710711:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829844.710718:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829844.710802:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829844.710807:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829844.710811:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787829844.838713:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829844.838722:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1787829844.838724:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787829844.838728:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787829844.838733:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787829844.838741:0:40798:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 4645 00000100:00080000:0.0:1787829844.838752:0:40798:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787829844.838755:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829844.838757:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787829844.838762:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787829844.838764:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787829844.838767:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:0.0:1787829844.838769:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:0.0:1787829844.838770:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829844.838771:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:0.0:1787829844.838775:0:40798:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787829844.838777:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829844.838779:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787829844.838782:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787829844.838784:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829844.838786:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1787829844.838788:0:40798:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829844.838797:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787829844.838799:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829844.838801:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787829844.838803:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787829844.838806:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829844.838815:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787829844.838822:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d25a40 x1874670646870400/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787829844.838830:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787829844.838833:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829844.838846:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787829844.838853:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829844.838856:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1787829844.838857:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787829844.838858:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787829844.838861:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787829844.838863:0:40798:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4645 00000100:00080000:0.0:1787829844.838866:0:40798:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787829844.838868:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829844.838870:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787829844.838872:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787829844.838873:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787829844.838875:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000020:00000001:0.0:1787829844.838877:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:0.0:1787829844.838878:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829844.838879:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:0.0:1787829844.838883:0:40798:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787829844.838883:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787829844.838888:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829844.838891:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:2.0:1787829844.838894:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 02000000:00000001:0.0:1787829844.838896:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787829844.838899:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829844.838900:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1787829844.838901:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:2.0:1787829844.838906:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000008:00100000:0.0:1787829844.838907:0:40798:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1787829844.838912:0:40798:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829844.838912:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829844.838915:0:40798:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829844.838922:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787829844.838923:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829844.838925:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787829844.838925:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787829844.838929:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787829844.838932:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829844.838933:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787829844.838936:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829844.838939:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787829844.838949:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d270c0 x1874670646870528/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787829844.838955:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d25a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646870400:192.168.203.143@tcp:250:kworker.0 00000100:00000001:0.0:1787829844.838968:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787829844.838972:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829844.838972:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:0.0:1787829844.838980:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829844.838981:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000100:00080000:0.0:1787829844.838983:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 02000000:00000001:2.0:1787829844.838986:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787829844.838987:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1787829844.838992:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787829844.838996:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 02000000:00000001:2.0:1787829844.838996:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:0.0:1787829844.839007:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787829844.839010:0:40798:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4645 02000000:00000001:2.0:1787829844.839012:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787829844.839013:0:40798:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787829844.839017:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829844.839021:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787829844.839023:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787829844.839024:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787829844.839026:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000020:00000001:0.0:1787829844.839027:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:0.0:1787829844.839029:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829844.839031:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:0.0:1787829844.839033:0:40798:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787829844.839034:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829844.839036:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787829844.839038:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787829844.839039:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 02000000:00000001:0.0:1787829844.839040:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829844.839042:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601343040 : -126234108208576 : ffff8d30d4d25a40) 00000100:00000001:0.0:1787829844.839043:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1787829844.839049:0:40798:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00000001:2.0:1787829844.839051:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000008:00000001:0.0:1787829844.839054:0:40798:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829844.839057:0:40798:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829844.839061:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787829844.839062:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829844.839063:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787829844.839065:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787829844.839066:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829844.839073:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787829844.839077:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d24000 x1874670646870656/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787829844.839082:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787829844.839084:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829844.839086:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787829844.839088:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829844.839090:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1787829844.839091:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787829844.839092:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787829844.839094:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787829844.839096:0:40798:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4645 00000100:00080000:0.0:1787829844.839099:0:40798:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000800:00000100:2.0:1787829844.839104:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:0.0:1787829844.839106:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829844.839108:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787829844.839110:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787829844.839111:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787829844.839113:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000020:00000001:0.0:1787829844.839115:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:0.0:1787829844.839117:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829844.839118:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:0.0:1787829844.839121:0:40798:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787829844.839123:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829844.839123:0:2390:0:(events.c:34:request_out_callback()) Process entered 02000000:00000001:0.0:1787829844.839125:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787829844.839129:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787829844.839131:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829844.839132:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787829844.839140:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1787829844.839142:0:40798:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00000001:2.0:1787829844.839142:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:0.0:1787829844.839153:0:40798:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829844.839155:0:40798:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829844.839155:0:2390:0:(events.c:68:request_out_callback()) Process leaving 02000000:00000001:0.0:1787829844.839161:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787829844.839163:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829844.839165:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787829844.839166:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:2.0:1787829844.839166:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829844.839170:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829844.839176:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829844.839176:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4299319040 00000100:00100000:0.0:1787829844.839185:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d24f00 x1874670646870784/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787829844.839191:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787829844.839193:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829844.839194:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1787829844.839195:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829844.839217:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4299319040 00000100:00000001:2.0:1787829844.839236:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 00000100:00000001:2.0:1787829844.839241:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787829844.839244:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829844.839248:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1787829844.839255:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829844/real 1787829844] req@ffff8d30d4d25a40 x1874670646870400/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829869 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1787829844.839267:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787829844.839269:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1787829844.839273:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1787829844.839277:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787829844.839278:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829844.839281:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787829844.839283:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787829844.839288:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d25a40 x1874670646870400/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829869 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787829844.839295:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829844.839299:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787829844.839302:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829844.839312:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787829844.839314:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829844.839317:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787829844.839320:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787829844.839323:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829844.839327:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:2.0:1787829844.839330:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1787829844.839334:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d25a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646870400:192.168.203.143@tcp:250:kworker.0 00000100:00000001:2.0:1787829844.839339:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829844.839341:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787829844.839343:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787829844.839346:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1787829844.839349:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787829844.839352:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:2.0:1787829844.839354:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:2.0:1787829844.839360:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787829844.839362:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1787829844.839366:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787829844.839368:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829844.839370:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829844.839378:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1787829844.839380:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829844.839383:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787829844.839385:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1787829844.839386:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1787829844.839387:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787829844.839389:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829844.839391:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787829844.839393:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787829844.839396:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787829844.839399:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d270c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646870528:192.168.203.143@tcp:38:kworker.0 00000100:00000001:2.0:1787829844.839403:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787829844.839405:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1787829844.839407:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829844.839408:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1787829844.839411:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829844.839417:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787829844.839418:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601348800 : -126234108202816 : ffff8d30d4d270c0) 00000100:00000001:2.0:1787829844.839423:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1787829844.839437:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:2.0:1787829844.839442:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1787829844.839444:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787829844.839445:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829844.839447:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1787829844.839449:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829844.839454:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4299317782 00000100:00000001:2.0:1787829844.839457:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4299319040 00000100:00000001:2.0:1787829844.839459:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4299317782 00000100:00000001:2.0:1787829844.839461:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4299319040 00000100:00000001:2.0:1787829844.839463:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787829844.839465:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829844.839466:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1787829844.839470:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829844/real 1787829844] req@ffff8d30d4d270c0 x1874670646870528/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829869 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1787829844.839480:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787829844.839481:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1787829844.839484:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1787829844.839486:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787829844.839487:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829844.839489:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787829844.839490:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787829844.839494:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d270c0 x1874670646870528/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829869 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787829844.839501:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829844.839503:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787829844.839505:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829844.839506:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787829844.839508:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829844.839510:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787829844.839512:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787829844.839514:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829844.839517:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:2.0:1787829844.839520:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1787829844.839523:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d270c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646870528:192.168.203.143@tcp:38:kworker.0 00000100:00000001:2.0:1787829844.839527:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1787829844.839529:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787829844.839530:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829844.839533:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787829844.839535:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787829844.839537:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787829844.839539:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d24000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646870656:192.168.203.143@tcp:38:kworker.0 00000100:00000001:2.0:1787829844.839543:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787829844.839545:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1787829844.839547:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829844.839548:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1787829844.839551:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829844.839556:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787829844.839557:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601336320 : -126234108215296 : ffff8d30d4d24000) 00000100:00000001:2.0:1787829844.839561:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1787829844.839571:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:2.0:1787829844.839575:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1787829844.839577:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787829844.839579:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829844.839580:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1787829844.839582:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829844.839586:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4299317782 00000100:00000001:2.0:1787829844.839588:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4299319040 00000100:00000001:2.0:1787829844.839590:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4299317782 00000100:00000001:2.0:1787829844.839592:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4299319040 00000100:00000001:2.0:1787829844.839594:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787829844.839596:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829844.839597:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1787829844.839600:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829844/real 1787829844] req@ffff8d30d4d24000 x1874670646870656/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829869 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1787829844.839621:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787829844.839622:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1787829844.839624:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1787829844.839625:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787829844.839626:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829844.839627:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787829844.839628:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787829844.839631:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d24000 x1874670646870656/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829869 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787829844.839635:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829844.839637:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787829844.839638:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829844.839640:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787829844.839640:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829844.839641:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787829844.839642:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787829844.839644:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829844.839646:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:2.0:1787829844.839647:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1787829844.839648:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d24000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646870656:192.168.203.143@tcp:38:kworker.0 00000100:00000001:2.0:1787829844.839650:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1787829844.839651:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787829844.839652:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829844.839653:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787829844.839654:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787829844.839655:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787829844.839657:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d24f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646870784:192.168.203.143@tcp:8:kworker.0 00000100:00000001:2.0:1787829844.839660:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787829844.839661:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1787829844.839662:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829844.839663:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1787829844.839665:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829844.839669:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787829844.839669:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601340160 : -126234108211456 : ffff8d30d4d24f00) 00000100:00000001:2.0:1787829844.839672:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1787829844.839679:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:2.0:1787829844.839682:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1787829844.839683:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787829844.839684:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829844.839685:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1787829844.839686:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829844.839689:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4299317796 00000100:00000001:2.0:1787829844.839690:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4299319040 00000100:00000001:2.0:1787829844.839692:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4299317796 00000100:00000001:2.0:1787829844.839694:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4299319040 00000100:00000001:2.0:1787829844.839695:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787829844.839696:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829844.839697:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1787829844.839699:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829844/real 1787829844] req@ffff8d30d4d24f00 x1874670646870784/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829869 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1787829844.839703:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787829844.839704:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1787829844.839705:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1787829844.839706:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787829844.839706:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829844.839707:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787829844.839708:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787829844.839710:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d24f00 x1874670646870784/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829869 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787829844.839714:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829844.839715:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787829844.839717:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829844.839718:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787829844.839718:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829844.839719:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787829844.839720:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787829844.839721:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829844.839723:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:2.0:1787829844.839724:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1787829844.839726:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d24f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646870784:192.168.203.143@tcp:8:kworker.0 00000100:00000001:2.0:1787829844.839727:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829844.839729:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787829844.839730:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787829844.839731:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1787829844.839732:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787829844.839733:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:2.0:1787829844.839734:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:2.0:1787829844.839736:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787829844.839737:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1787829844.839740:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787829844.839740:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829844.839741:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787829844.839742:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787829844.839743:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1787829844.839744:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787829844.839745:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:2.0:1787829844.839745:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:2.0:1787829844.839746:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787829844.839747:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1787829844.839749:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787829844.839749:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829844.839750:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787829844.839751:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787829844.839751:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1787829844.839752:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787829844.839753:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:2.0:1787829844.839753:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:2.0:1787829844.839754:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787829844.839755:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1787829844.839756:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787829844.839756:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829844.839757:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829844.839759:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1787829844.839760:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829844.839761:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787829844.839762:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829845.734689:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829845.734695:0:815:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1787829845.734697:0:815:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1787829845.734699:0:815:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829845.734703:0:815:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1787829845.734705:0:815:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829845.734706:0:815:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829845.734708:0:815:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829845.734761:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829845.734765:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829845.734768:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829846.758716:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829846.758723:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829846.758772:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829846.758776:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829846.758778:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829847.782708:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829847.782713:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829847.782760:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829847.782764:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829847.782766:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829848.806695:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829848.806700:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829848.806752:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829848.806755:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829848.806758:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829849.830686:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829849.830693:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829849.830740:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829849.830744:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829849.830748:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787829849.958728:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829849.958738:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1787829849.958740:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787829849.958743:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787829849.958747:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787829849.958754:0:40798:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 4650 00000100:00080000:0.0:1787829849.958765:0:40798:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787829849.958768:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829849.958771:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787829849.958775:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787829849.958778:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787829849.958781:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:0.0:1787829849.958783:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:0.0:1787829849.958786:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829849.958787:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:0.0:1787829849.958792:0:40798:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787829849.958795:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829849.958798:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787829849.958802:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787829849.958804:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829849.958806:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1787829849.958808:0:40798:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829849.958827:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787829849.958829:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829849.958832:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787829849.958834:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787829849.958838:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829849.958851:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787829849.958861:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d24f00 x1874670646870912/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787829849.958872:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787829849.958876:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829849.958889:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787829849.958891:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829849.958894:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1787829849.958895:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787829849.958896:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787829849.958898:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787829849.958900:0:40798:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4650 00000100:00080000:0.0:1787829849.958904:0:40798:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787829849.958905:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829849.958906:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787829849.958908:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787829849.958909:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787829849.958912:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000020:00000001:0.0:1787829849.958915:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:0.0:1787829849.958917:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829849.958919:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:0.0:1787829849.958923:0:40798:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787829849.958927:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829849.958928:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:2.0:1787829849.958931:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:0.0:1787829849.958935:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787829849.958937:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 02000000:00000001:0.0:1787829849.958939:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829849.958942:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1787829849.958944:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:2.0:1787829849.958947:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000008:00100000:0.0:1787829849.958950:0:40798:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00000001:2.0:1787829849.958951:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:0.0:1787829849.958957:0:40798:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829849.958958:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:0.0:1787829849.958960:0:40798:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829849.958963:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787829849.958965:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787829849.958969:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829849.958969:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829849.958970:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787829849.958975:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787829849.958977:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787829849.958980:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d24f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646870912:192.168.203.143@tcp:250:kworker.0 00000100:00000001:0.0:1787829849.958982:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829849.958989:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00100000:0.0:1787829849.958990:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d24000 x1874670646871040/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:2.0:1787829849.958994:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:0.0:1787829849.958999:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 02000000:00000001:2.0:1787829849.958999:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829849.959003:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829849.959005:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:0.0:1787829849.959006:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787829849.959010:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 02000000:00000001:2.0:1787829849.959012:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787829849.959013:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1787829849.959016:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787829849.959019:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787829849.959021:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787829849.959023:0:40798:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4650 00000100:00080000:0.0:1787829849.959026:0:40798:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787829849.959027:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829849.959028:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787829849.959030:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787829849.959031:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:2.0:1787829849.959032:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000020:00000001:0.0:1787829849.959033:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:2.0:1787829849.959036:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601340160 : -126234108211456 : ffff8d30d4d24f00) 00000020:00000001:0.0:1787829849.959037:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:0.0:1787829849.959043:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829849.959045:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:2.0:1787829849.959045:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00080000:0.0:1787829849.959049:0:40798:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787829849.959052:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829849.959053:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787829849.959055:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787829849.959057:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829849.959058:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1787829849.959060:0:40798:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1787829849.959062:0:40798:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829849.959063:0:40798:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829849.959066:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787829849.959067:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829849.959068:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787829849.959069:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787829849.959070:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829849.959073:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787829849.959077:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d270c0 x1874670646871168/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787829849.959086:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787829849.959088:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829849.959091:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:2.0:1787829849.959092:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00080000:0.0:1787829849.959094:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829849.959098:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1787829849.959099:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787829849.959100:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787829849.959101:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787829849.959104:0:40798:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4650 00000100:00080000:0.0:1787829849.959106:0:40798:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787829849.959108:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829849.959109:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:2.0:1787829849.959110:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000020:00000001:0.0:1787829849.959111:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787829849.959114:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787829849.959115:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000020:00000001:0.0:1787829849.959118:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:2.0:1787829849.959118:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787829849.959120:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829849.959123:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:2.0:1787829849.959123:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787829849.959130:0:40798:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787829849.959130:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787829849.959134:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829849.959135:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787829849.959137:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787829849.959138:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829849.959139:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829849.959144:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1787829849.959146:0:40798:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1787829849.959148:0:40798:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829849.959148:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4299324160 00000100:00000001:0.0:1787829849.959150:0:40798:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829849.959156:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787829849.959156:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829849.959158:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787829849.959159:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787829849.959160:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829849.959161:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1787829849.959164:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787829849.959169:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d25a40 x1874670646871296/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787829849.959170:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4299324160 00000100:00000001:0.0:1787829849.959179:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787829849.959181:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829849.959181:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1787829849.959184:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829849.959200:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787829849.959206:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829849.959208:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1787829849.959216:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829849/real 1787829849] req@ffff8d30d4d24f00 x1874670646870912/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829874 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1787829849.959228:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787829849.959231:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1787829849.959235:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1787829849.959238:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787829849.959240:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829849.959243:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787829849.959245:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787829849.959250:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d24f00 x1874670646870912/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829874 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787829849.959258:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829849.959262:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787829849.959265:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829849.959267:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787829849.959268:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829849.959270:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787829849.959272:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787829849.959275:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829849.959279:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:2.0:1787829849.959282:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1787829849.959286:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d24f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646870912:192.168.203.143@tcp:250:kworker.0 00000100:00000001:2.0:1787829849.959291:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829849.959293:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787829849.959295:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787829849.959297:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1787829849.959300:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787829849.959303:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:2.0:1787829849.959304:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:2.0:1787829849.959308:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787829849.959311:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1787829849.959315:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787829849.959317:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829849.959319:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829849.959327:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1787829849.959329:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829849.959331:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787829849.959332:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1787829849.959334:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1787829849.959335:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787829849.959337:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829849.959339:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787829849.959341:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787829849.959343:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787829849.959346:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d24000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646871040:192.168.203.143@tcp:38:kworker.0 00000100:00000001:2.0:1787829849.959350:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787829849.959352:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1787829849.959354:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829849.959356:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1787829849.959359:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829849.959366:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787829849.959367:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601336320 : -126234108215296 : ffff8d30d4d24000) 00000100:00000001:2.0:1787829849.959372:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1787829849.959388:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:2.0:1787829849.959394:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1787829849.959396:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787829849.959398:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829849.959399:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1787829849.959402:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829849.959407:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4299322902 00000100:00000001:2.0:1787829849.959410:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4299324160 00000100:00000001:2.0:1787829849.959413:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4299322902 00000100:00000001:2.0:1787829849.959415:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4299324160 00000100:00000001:2.0:1787829849.959417:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787829849.959420:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829849.959422:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1787829849.959427:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829849/real 1787829849] req@ffff8d30d4d24000 x1874670646871040/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829874 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1787829849.959437:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787829849.959438:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1787829849.959441:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1787829849.959443:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787829849.959444:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829849.959447:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787829849.959448:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787829849.959453:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d24000 x1874670646871040/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829874 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787829849.959461:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829849.959463:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787829849.959466:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829849.959468:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787829849.959469:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829849.959471:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787829849.959473:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787829849.959476:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829849.959480:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:2.0:1787829849.959482:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1787829849.959486:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d24000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646871040:192.168.203.143@tcp:38:kworker.0 00000100:00000001:2.0:1787829849.959490:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1787829849.959491:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787829849.959493:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829849.959495:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787829849.959497:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787829849.959499:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787829849.959502:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d270c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646871168:192.168.203.143@tcp:38:kworker.0 00000100:00000001:2.0:1787829849.959508:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787829849.959509:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1787829849.959511:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829849.959512:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1787829849.959514:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829849.959519:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787829849.959520:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601348800 : -126234108202816 : ffff8d30d4d270c0) 00000100:00000001:2.0:1787829849.959523:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1787829849.959533:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:2.0:1787829849.959536:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1787829849.959538:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787829849.959539:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829849.959540:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1787829849.959542:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829849.959545:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4299322902 00000100:00000001:2.0:1787829849.959547:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4299324160 00000100:00000001:2.0:1787829849.959548:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4299322902 00000100:00000001:2.0:1787829849.959550:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4299324160 00000100:00000001:2.0:1787829849.959551:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787829849.959553:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829849.959554:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1787829849.959557:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829849/real 1787829849] req@ffff8d30d4d270c0 x1874670646871168/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829874 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1787829849.959562:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787829849.959562:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1787829849.959564:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1787829849.959565:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787829849.959566:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829849.959567:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787829849.959568:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787829849.959571:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d270c0 x1874670646871168/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829874 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787829849.959581:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829849.959583:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787829849.959586:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829849.959587:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787829849.959589:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829849.959591:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787829849.959593:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787829849.959595:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829849.959599:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:2.0:1787829849.959628:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1787829849.959632:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d270c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646871168:192.168.203.143@tcp:38:kworker.0 00000100:00000001:2.0:1787829849.959637:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1787829849.959638:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787829849.959640:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829849.959642:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787829849.959644:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787829849.959646:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787829849.959649:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d25a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646871296:192.168.203.143@tcp:8:kworker.0 00000100:00000001:2.0:1787829849.959652:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787829849.959655:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1787829849.959656:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829849.959658:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1787829849.959661:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829849.959667:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787829849.959669:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601343040 : -126234108208576 : ffff8d30d4d25a40) 00000100:00000001:2.0:1787829849.959672:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1787829849.959685:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:2.0:1787829849.959691:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1787829849.959693:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787829849.959694:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829849.959697:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1787829849.959699:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829849.959702:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4299322916 00000100:00000001:2.0:1787829849.959705:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4299324160 00000100:00000001:2.0:1787829849.959707:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4299322916 00000100:00000001:2.0:1787829849.959709:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4299324160 00000100:00000001:2.0:1787829849.959711:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787829849.959713:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829849.959715:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1787829849.959719:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829849/real 1787829849] req@ffff8d30d4d25a40 x1874670646871296/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829874 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1787829849.959727:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787829849.959728:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1787829849.959741:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1787829849.959743:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787829849.959745:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829849.959746:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787829849.959748:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787829849.959752:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d25a40 x1874670646871296/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829874 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787829849.959760:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829849.959762:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787829849.959764:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829849.959766:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787829849.959767:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829849.959769:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787829849.959771:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787829849.959773:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829849.959777:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:2.0:1787829849.959779:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1787829849.959783:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d25a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646871296:192.168.203.143@tcp:8:kworker.0 00000100:00000001:2.0:1787829849.959787:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829849.959789:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787829849.959791:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787829849.959792:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1787829849.959794:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787829849.959796:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:2.0:1787829849.959797:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:2.0:1787829849.959800:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787829849.959801:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1787829849.959803:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787829849.959804:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829849.959806:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787829849.959807:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787829849.959809:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1787829849.959810:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787829849.959812:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:2.0:1787829849.959813:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:2.0:1787829849.959814:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787829849.959816:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1787829849.959819:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787829849.959820:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829849.959822:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787829849.959823:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787829849.959824:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1787829849.959826:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787829849.959827:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:2.0:1787829849.959828:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:2.0:1787829849.959830:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787829849.959831:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1787829849.959833:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787829849.959834:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829849.959836:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829849.959840:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1787829849.959841:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829849.959844:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787829849.959845:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829850.854716:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829850.854722:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829850.854772:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829850.854776:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829850.854778:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829851.878671:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829851.878677:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829851.878723:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829851.878726:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829851.878728:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829852.902698:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829852.902703:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829852.902763:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829852.902768:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829852.902772:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829853.926697:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829853.926709:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829853.926753:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829853.926757:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829853.926759:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829854.950694:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829854.950699:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829854.950746:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829854.950749:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829854.950751:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787829855.078716:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829855.078725:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1787829855.078727:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787829855.078731:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787829855.078736:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787829855.078744:0:40701:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 4655 00000100:00080000:3.0:1787829855.078754:0:40701:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787829855.078758:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829855.078760:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:3.0:1787829855.078765:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1787829855.078767:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1787829855.078771:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:3.0:1787829855.078773:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:3.0:1787829855.078776:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829855.078777:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:3.0:1787829855.078782:0:40701:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:3.0:1787829855.078785:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829855.078788:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787829855.078791:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829855.078795:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829855.078797:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:3.0:1787829855.078800:0:40701:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829855.078811:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787829855.078813:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829855.078816:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787829855.078819:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787829855.078823:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829855.078836:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829855.078845:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c7015a40 x1874670646871424/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787829855.078857:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787829855.078862:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829855.078879:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787829855.078882:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829855.078886:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1787829855.078887:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787829855.078889:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787829855.078892:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787829855.078895:0:40701:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4655 00000100:00080000:3.0:1787829855.078899:0:40701:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787829855.078902:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829855.078903:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:3.0:1787829855.078905:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1787829855.078907:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1787829855.078910:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000020:00000001:3.0:1787829855.078913:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:3.0:1787829855.078915:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829855.078917:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:3.0:1787829855.078921:0:40701:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:3.0:1787829855.078924:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829855.078925:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:2.0:1787829855.078929:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:3.0:1787829855.078932:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787829855.078935:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 02000000:00000001:3.0:1787829855.078936:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829855.078940:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787829855.078940:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:2.0:1787829855.078945:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000008:00100000:3.0:1787829855.078947:0:40701:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00000001:2.0:1787829855.078949:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:3.0:1787829855.078951:0:40701:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829855.078954:0:40701:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829855.078956:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787829855.078958:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787829855.078960:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829855.078961:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829855.078965:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787829855.078968:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829855.078968:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787829855.078971:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829855.078976:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787829855.078978:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c7015a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646871424:192.168.203.143@tcp:250:kworker.0 00000100:00100000:3.0:1787829855.078983:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c7016580 x1874670646871552/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787829855.078988:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:3.0:1787829855.078992:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 02000000:00000001:2.0:1787829855.078995:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:3.0:1787829855.078995:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829855.079000:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829855.079001:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787829855.079003:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 02000000:00000001:2.0:1787829855.079008:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00080000:3.0:1787829855.079008:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1787829855.079009:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787829855.079013:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787829855.079015:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787829855.079017:0:40701:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4655 02000000:00000001:2.0:1787829855.079018:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787829855.079020:0:40701:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787829855.079023:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829855.079024:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:3.0:1787829855.079026:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1787829855.079027:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1787829855.079029:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000020:00000001:3.0:1787829855.079030:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:3.0:1787829855.079032:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829855.079033:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:3.0:1787829855.079035:0:40701:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:3.0:1787829855.079036:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829855.079038:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 02000000:00000001:3.0:1787829855.079038:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787829855.079042:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787829855.079043:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369542208 : -126234340009408 : ffff8d30c7015a40) 02000000:00000001:3.0:1787829855.079044:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829855.079048:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:3.0:1787829855.079050:0:40701:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00000001:2.0:1787829855.079052:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000008:00000001:3.0:1787829855.079052:0:40701:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829855.079056:0:40701:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829855.079060:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787829855.079060:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829855.079061:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787829855.079062:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787829855.079063:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829855.079067:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829855.079070:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c7015680 x1874670646871680/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787829855.079075:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787829855.079077:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829855.079079:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787829855.079080:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829855.079083:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1787829855.079085:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787829855.079086:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787829855.079087:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787829855.079090:0:40701:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4655 00000100:00080000:3.0:1787829855.079093:0:40701:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787829855.079096:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000800:00000100:2.0:1787829855.079098:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:3.0:1787829855.079099:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:3.0:1787829855.079104:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1787829855.079105:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1787829855.079106:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000020:00000001:3.0:1787829855.079108:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:3.0:1787829855.079110:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829855.079112:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:2.0:1787829855.079117:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00080000:3.0:1787829855.079117:0:40701:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:3.0:1787829855.079122:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829855.079123:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:2.0:1787829855.079125:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 02000000:00000001:3.0:1787829855.079126:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829855.079130:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829855.079132:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829855.079132:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:3.0:1787829855.079135:0:40701:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00000001:2.0:1787829855.079138:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000008:00000001:3.0:1787829855.079138:0:40701:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829855.079142:0:40701:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829855.079146:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787829855.079147:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829855.079150:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829855.079160:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787829855.079163:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787829855.079165:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829855.079171:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829855.079172:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4299329280 00000100:00100000:3.0:1787829855.079177:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c7014f00 x1874670646871808/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787829855.079186:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 00000100:00000001:2.0:1787829855.079197:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4299329280 00000100:00000001:3.0:1787829855.079197:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787829855.079200:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829855.079205:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 00000100:00000001:3.0:1787829855.079206:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829855.079213:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787829855.079217:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829855.079221:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1787829855.079228:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829855/real 1787829855] req@ffff8d30c7015a40 x1874670646871424/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829880 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1787829855.079239:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787829855.079241:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1787829855.079245:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1787829855.079248:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787829855.079250:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829855.079252:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787829855.079255:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787829855.079259:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c7015a40 x1874670646871424/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829880 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787829855.079266:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829855.079270:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787829855.079272:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829855.079274:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787829855.079276:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829855.079279:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787829855.079291:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787829855.079295:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829855.079299:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:2.0:1787829855.079302:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1787829855.079305:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c7015a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646871424:192.168.203.143@tcp:250:kworker.0 00000100:00000001:2.0:1787829855.079310:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829855.079313:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787829855.079314:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787829855.079317:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1787829855.079320:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787829855.079322:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:2.0:1787829855.079324:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:2.0:1787829855.079329:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787829855.079330:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1787829855.079333:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787829855.079335:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829855.079337:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829855.079344:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1787829855.079346:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829855.079348:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787829855.079350:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1787829855.079351:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1787829855.079352:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787829855.079353:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829855.079355:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787829855.079357:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787829855.079359:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787829855.079363:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c7016580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646871552:192.168.203.143@tcp:38:kworker.0 00000100:00000001:2.0:1787829855.079367:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787829855.079369:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1787829855.079370:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829855.079372:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1787829855.079375:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829855.079382:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787829855.079383:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369545088 : -126234340006528 : ffff8d30c7016580) 00000100:00000001:2.0:1787829855.079388:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1787829855.079404:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:2.0:1787829855.079411:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1787829855.079414:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787829855.079415:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829855.079417:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1787829855.079421:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829855.079426:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4299328022 00000100:00000001:2.0:1787829855.079429:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4299329280 00000100:00000001:2.0:1787829855.079432:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4299328022 00000100:00000001:2.0:1787829855.079434:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4299329280 00000100:00000001:2.0:1787829855.079436:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787829855.079439:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829855.079441:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1787829855.079446:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829855/real 1787829855] req@ffff8d30c7016580 x1874670646871552/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829880 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1787829855.079456:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787829855.079457:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1787829855.079461:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1787829855.079463:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787829855.079465:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829855.079467:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787829855.079468:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787829855.079473:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c7016580 x1874670646871552/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829880 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787829855.079481:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829855.079483:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787829855.079486:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829855.079488:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787829855.079489:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829855.079491:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787829855.079493:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787829855.079495:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829855.079499:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:2.0:1787829855.079501:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1787829855.079505:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c7016580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646871552:192.168.203.143@tcp:38:kworker.0 00000100:00000001:2.0:1787829855.079509:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1787829855.079510:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787829855.079512:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829855.079514:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787829855.079516:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787829855.079518:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787829855.079521:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c7015680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646871680:192.168.203.143@tcp:38:kworker.0 00000100:00000001:2.0:1787829855.079524:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787829855.079526:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1787829855.079528:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829855.079530:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1787829855.079532:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829855.079537:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787829855.079539:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369541248 : -126234340010368 : ffff8d30c7015680) 00000100:00000001:2.0:1787829855.079542:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1787829855.079565:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:2.0:1787829855.079571:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1787829855.079575:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787829855.079576:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829855.079578:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1787829855.079581:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829855.079586:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4299328022 00000100:00000001:2.0:1787829855.079588:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4299329280 00000100:00000001:2.0:1787829855.079590:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4299328022 00000100:00000001:2.0:1787829855.079592:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4299329280 00000100:00000001:2.0:1787829855.079595:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787829855.079597:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829855.079599:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1787829855.079632:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829855/real 1787829855] req@ffff8d30c7015680 x1874670646871680/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829880 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1787829855.079641:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787829855.079643:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1787829855.079647:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1787829855.079649:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787829855.079650:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829855.079652:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787829855.079654:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787829855.079658:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c7015680 x1874670646871680/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829880 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787829855.079665:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829855.079668:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787829855.079670:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829855.079672:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787829855.079673:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829855.079675:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787829855.079676:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787829855.079679:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829855.079682:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:2.0:1787829855.079685:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1787829855.079688:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c7015680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646871680:192.168.203.143@tcp:38:kworker.0 00000100:00000001:2.0:1787829855.079692:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1787829855.079694:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787829855.079695:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829855.079697:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787829855.079699:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787829855.079702:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787829855.079704:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c7014f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646871808:192.168.203.143@tcp:8:kworker.0 00000100:00000001:2.0:1787829855.079709:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787829855.079710:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1787829855.079712:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829855.079713:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1787829855.079717:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829855.079723:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787829855.079725:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369539328 : -126234340012288 : ffff8d30c7014f00) 00000100:00000001:2.0:1787829855.079729:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1787829855.079743:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:2.0:1787829855.079748:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1787829855.079751:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787829855.079753:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829855.079754:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1787829855.079757:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829855.079761:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4299328036 00000100:00000001:2.0:1787829855.079763:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4299329280 00000100:00000001:2.0:1787829855.079766:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4299328036 00000100:00000001:2.0:1787829855.079769:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4299329280 00000100:00000001:2.0:1787829855.079771:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787829855.079773:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829855.079775:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1787829855.079780:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829855/real 1787829855] req@ffff8d30c7014f00 x1874670646871808/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829880 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1787829855.079789:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787829855.079790:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1787829855.079793:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1787829855.079795:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787829855.079797:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829855.079799:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787829855.079801:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787829855.079805:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c7014f00 x1874670646871808/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829880 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787829855.079813:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829855.079816:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787829855.079818:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829855.079820:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787829855.079821:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829855.079823:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787829855.079825:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787829855.079828:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829855.079832:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:2.0:1787829855.079834:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1787829855.079837:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c7014f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646871808:192.168.203.143@tcp:8:kworker.0 00000100:00000001:2.0:1787829855.079840:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829855.079842:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787829855.079843:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787829855.079844:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1787829855.079846:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787829855.079847:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:2.0:1787829855.079848:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:2.0:1787829855.079850:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787829855.079851:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1787829855.079853:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787829855.079855:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829855.079857:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787829855.079858:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787829855.079859:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1787829855.079861:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787829855.079862:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:2.0:1787829855.079864:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:2.0:1787829855.079865:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787829855.079866:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1787829855.079868:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787829855.079869:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829855.079871:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787829855.079872:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787829855.079874:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1787829855.079875:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787829855.079877:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:2.0:1787829855.079878:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:2.0:1787829855.079879:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787829855.079880:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1787829855.079883:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787829855.079884:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829855.079886:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829855.079890:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1787829855.079891:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829855.079893:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787829855.079894:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829855.974675:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829855.974683:0:815:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1787829855.974684:0:815:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1787829855.974686:0:815:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829855.974691:0:815:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1787829855.974693:0:815:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829855.974694:0:815:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829855.974696:0:815:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829855.974749:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829855.974754:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829855.974758:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829856.998721:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829856.998729:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829856.998828:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829856.998832:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829856.998909:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829858.022705:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829858.022713:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829858.022803:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829858.022809:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829858.022814:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829859.046681:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829859.046686:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829859.046739:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829859.046743:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829859.046745:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829860.070722:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829860.070728:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829860.070788:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829860.070792:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829860.070796:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787829860.198699:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829860.198707:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1787829860.198709:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787829860.198712:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787829860.198716:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787829860.198722:0:40701:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 4660 00000100:00080000:3.0:1787829860.198732:0:40701:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787829860.198734:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829860.198736:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:3.0:1787829860.198739:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1787829860.198741:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1787829860.198743:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:3.0:1787829860.198745:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:3.0:1787829860.198746:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829860.198747:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:3.0:1787829860.198751:0:40701:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:3.0:1787829860.198752:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829860.198754:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787829860.198757:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829860.198759:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829860.198760:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:3.0:1787829860.198763:0:40701:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829860.198772:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787829860.198773:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829860.198776:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787829860.198778:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787829860.198782:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829860.198792:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829860.198799:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c7014b40 x1874670646871936/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787829860.198810:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787829860.198814:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829860.198825:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787829860.198829:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829860.198835:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1787829860.198837:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787829860.198839:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787829860.198842:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787829860.198845:0:40701:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4660 00000100:00080000:3.0:1787829860.198848:0:40701:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787829860.198850:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829860.198852:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:3.0:1787829860.198854:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1787829860.198855:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1787829860.198857:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000020:00000001:3.0:1787829860.198860:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:3.0:1787829860.198862:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829860.198863:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:2.0:1787829860.198864:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00080000:3.0:1787829860.198866:0:40701:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787829860.198868:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787829860.198869:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829860.198871:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 02000000:00000001:3.0:1787829860.198871:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:2.0:1787829860.198873:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787829860.198875:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829860.198876:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829860.198878:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829860.198879:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:3.0:1787829860.198880:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 02000000:00000001:2.0:1787829860.198882:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787829860.198884:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000008:00100000:3.0:1787829860.198885:0:40701:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1787829860.198888:0:40701:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787829860.198889:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c7014b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646871936:192.168.203.143@tcp:250:kworker.0 00000100:00000001:3.0:1787829860.198890:0:40701:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829860.198894:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1787829860.198895:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787829860.198896:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829860.198898:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:3.0:1787829860.198898:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787829860.198899:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787829860.198901:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829860.198901:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829860.198906:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829860.198908:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00100000:3.0:1787829860.198911:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c70161c0 x1874670646872064/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:2.0:1787829860.198912:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829860.198918:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787829860.198920:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829860.198922:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787829860.198925:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:2.0:1787829860.198926:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787829860.198928:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369538368 : -126234340013248 : ffff8d30c7014b40) 00000100:00080000:3.0:1787829860.198928:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1787829860.198930:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:2.0:1787829860.198932:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00080000:3.0:1787829860.198932:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787829860.198934:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787829860.198937:0:40701:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4660 00000100:00080000:3.0:1787829860.198950:0:40701:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787829860.198953:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829860.198954:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:3.0:1787829860.198956:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1787829860.198958:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1787829860.198959:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000020:00000001:3.0:1787829860.198961:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:3.0:1787829860.198963:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829860.198965:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000800:00000100:2.0:1787829860.198967:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00080000:3.0:1787829860.198968:0:40701:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:3.0:1787829860.198971:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829860.198973:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787829860.198976:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829860.198977:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829860.198979:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:3.0:1787829860.198982:0:40701:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1787829860.198984:0:40701:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829860.198986:0:40701:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829860.198987:0:2390:0:(events.c:34:request_out_callback()) Process entered 02000000:00000001:3.0:1787829860.198989:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:2.0:1787829860.198991:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 02000000:00000001:3.0:1787829860.198991:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829860.198992:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:2.0:1787829860.198993:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829860.198994:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:2.0:1787829860.198995:0:2390:0:(events.c:68:request_out_callback()) Process leaving 02000000:00000001:3.0:1787829860.198996:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829860.198999:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829860.199001:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829860.199004:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c70152c0 x1874670646872192/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787829860.199005:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4299334400 00000100:00000001:3.0:1787829860.199011:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787829860.199013:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829860.199016:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787829860.199018:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:2.0:1787829860.199020:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 00000100:00080000:3.0:1787829860.199022:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1787829860.199023:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787829860.199025:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787829860.199027:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787829860.199030:0:40701:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4660 00000100:00080000:3.0:1787829860.199033:0:40701:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787829860.199035:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4299334400 00000100:00000001:3.0:1787829860.199036:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829860.199037:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:2.0:1787829860.199039:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 00000020:00000001:3.0:1787829860.199040:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1787829860.199041:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:2.0:1787829860.199043:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000020:00000001:3.0:1787829860.199043:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000020:00000001:3.0:1787829860.199045:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:2.0:1787829860.199046:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829860.199048:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829860.199049:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00000001:3.0:1787829860.199050:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:3.0:1787829860.199054:0:40701:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00100000:2.0:1787829860.199055:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829860/real 1787829860] req@ffff8d30c7014b40 x1874670646871936/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829885 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829860.199057:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829860.199059:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787829860.199062:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829860.199064:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:2.0:1787829860.199066:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787829860.199067:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829860.199067:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:3.0:1787829860.199070:0:40701:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00000001:2.0:1787829860.199071:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000008:00000001:3.0:1787829860.199072:0:40701:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829860.199073:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829860.199074:0:40701:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829860.199075:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829860.199077:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 02000000:00000001:3.0:1787829860.199077:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787829860.199078:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829860.199079:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829860.199080:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787829860.199082:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00100000:2.0:1787829860.199084:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c7014b40 x1874670646871936/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829885 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1787829860.199084:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829860.199088:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829860.199092:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c70143c0 x1874670646872320/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787829860.199099:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829860.199099:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787829860.199101:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829860.199103:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829860.199103:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829860.199106:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829860.199108:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787829860.199109:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829860.199111:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787829860.199113:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787829860.199129:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829860.199134:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:2.0:1787829860.199136:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1787829860.199140:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c7014b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646871936:192.168.203.143@tcp:250:kworker.0 00000100:00000001:2.0:1787829860.199145:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829860.199147:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787829860.199149:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787829860.199151:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1787829860.199155:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787829860.199157:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:2.0:1787829860.199159:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:2.0:1787829860.199162:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787829860.199164:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1787829860.199166:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787829860.199172:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829860.199174:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829860.199180:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1787829860.199181:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829860.199183:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787829860.199184:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1787829860.199185:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1787829860.199186:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787829860.199187:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829860.199188:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787829860.199190:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787829860.199191:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787829860.199193:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c70161c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646872064:192.168.203.143@tcp:38:kworker.0 00000100:00000001:2.0:1787829860.199196:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787829860.199198:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1787829860.199199:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829860.199202:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1787829860.199205:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829860.199212:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787829860.199213:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369544128 : -126234340007488 : ffff8d30c70161c0) 00000100:00000001:2.0:1787829860.199217:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1787829860.199231:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:2.0:1787829860.199237:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1787829860.199240:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787829860.199241:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829860.199243:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1787829860.199246:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829860.199250:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4299333142 00000100:00000001:2.0:1787829860.199253:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4299334400 00000100:00000001:2.0:1787829860.199256:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4299333142 00000100:00000001:2.0:1787829860.199258:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4299334400 00000100:00000001:2.0:1787829860.199259:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787829860.199261:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829860.199262:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1787829860.199266:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829860/real 1787829860] req@ffff8d30c70161c0 x1874670646872064/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829885 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1787829860.199272:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787829860.199273:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1787829860.199275:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1787829860.199276:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787829860.199277:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829860.199278:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787829860.199279:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787829860.199282:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c70161c0 x1874670646872064/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829885 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787829860.199286:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829860.199287:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787829860.199289:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829860.199290:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787829860.199290:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829860.199292:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787829860.199293:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787829860.199295:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829860.199297:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:2.0:1787829860.199299:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1787829860.199303:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c70161c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646872064:192.168.203.143@tcp:38:kworker.0 00000100:00000001:2.0:1787829860.199306:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1787829860.199308:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787829860.199309:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829860.199311:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787829860.199313:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787829860.199315:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787829860.199318:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c70152c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646872192:192.168.203.143@tcp:38:kworker.0 00000100:00000001:2.0:1787829860.199325:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787829860.199326:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1787829860.199328:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829860.199330:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1787829860.199334:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829860.199339:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787829860.199341:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369540288 : -126234340011328 : ffff8d30c70152c0) 00000100:00000001:2.0:1787829860.199345:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1787829860.199356:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:2.0:1787829860.199360:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1787829860.199361:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787829860.199362:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829860.199363:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1787829860.199365:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829860.199368:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4299333142 00000100:00000001:2.0:1787829860.199369:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4299334400 00000100:00000001:2.0:1787829860.199371:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4299333142 00000100:00000001:2.0:1787829860.199373:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4299334400 00000100:00000001:2.0:1787829860.199374:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787829860.199375:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829860.199376:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1787829860.199379:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829860/real 1787829860] req@ffff8d30c70152c0 x1874670646872192/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829885 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1787829860.199387:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787829860.199389:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1787829860.199391:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1787829860.199392:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787829860.199394:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829860.199396:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787829860.199397:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787829860.199401:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c70152c0 x1874670646872192/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829885 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787829860.199416:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829860.199418:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787829860.199421:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829860.199423:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787829860.199424:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829860.199426:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787829860.199428:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787829860.199430:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829860.199433:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:2.0:1787829860.199434:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1787829860.199436:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c70152c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646872192:192.168.203.143@tcp:38:kworker.0 00000100:00000001:2.0:1787829860.199439:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1787829860.199440:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787829860.199441:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829860.199443:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787829860.199444:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787829860.199446:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787829860.199448:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c70143c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646872320:192.168.203.143@tcp:8:kworker.0 00000100:00000001:2.0:1787829860.199449:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787829860.199451:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1787829860.199452:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829860.199453:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1787829860.199456:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829860.199462:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787829860.199464:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369536448 : -126234340015168 : ffff8d30c70143c0) 00000100:00000001:2.0:1787829860.199467:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1787829860.199479:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:2.0:1787829860.199484:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1787829860.199487:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787829860.199488:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829860.199491:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1787829860.199493:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829860.199497:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4299333155 00000100:00000001:2.0:1787829860.199500:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4299334400 00000100:00000001:2.0:1787829860.199502:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4299333155 00000100:00000001:2.0:1787829860.199504:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4299334400 00000100:00000001:2.0:1787829860.199507:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787829860.199508:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829860.199510:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1787829860.199513:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829860/real 1787829860] req@ffff8d30c70143c0 x1874670646872320/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829885 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1787829860.199519:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787829860.199520:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1787829860.199522:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1787829860.199524:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787829860.199525:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829860.199526:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787829860.199527:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787829860.199529:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c70143c0 x1874670646872320/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829885 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787829860.199534:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829860.199535:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787829860.199536:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829860.199537:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787829860.199538:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829860.199539:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787829860.199540:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787829860.199542:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829860.199545:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:2.0:1787829860.199546:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1787829860.199549:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c70143c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646872320:192.168.203.143@tcp:8:kworker.0 00000100:00000001:2.0:1787829860.199551:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829860.199553:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787829860.199554:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787829860.199555:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1787829860.199556:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787829860.199557:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:2.0:1787829860.199566:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:2.0:1787829860.199569:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787829860.199570:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1787829860.199572:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787829860.199573:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829860.199575:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787829860.199576:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787829860.199578:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1787829860.199579:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787829860.199581:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:2.0:1787829860.199582:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:2.0:1787829860.199583:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787829860.199585:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1787829860.199587:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787829860.199588:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829860.199589:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787829860.199591:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787829860.199592:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1787829860.199594:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787829860.199595:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:2.0:1787829860.199596:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:2.0:1787829860.199598:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787829860.199599:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1787829860.199627:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787829860.199628:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829860.199630:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829860.199633:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1787829860.199634:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829860.199635:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787829860.199636:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829861.094685:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829861.094690:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829861.094744:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829861.094747:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829861.094750:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829862.118712:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829862.118717:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829862.118772:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829862.118776:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829862.118779:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829863.142698:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829863.142703:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829863.142761:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829863.142765:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829863.142767:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829864.166712:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829864.166722:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829864.166767:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829864.166772:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829864.166776:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829865.190701:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829865.190706:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829865.190759:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829865.190762:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829865.190765:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787829865.318739:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829865.318746:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1787829865.318748:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787829865.318752:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787829865.318755:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787829865.318761:0:40701:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 4665 00000100:00080000:3.0:1787829865.318770:0:40701:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787829865.318773:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829865.318775:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:3.0:1787829865.318778:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1787829865.318780:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1787829865.318783:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:3.0:1787829865.318786:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:3.0:1787829865.318787:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829865.318789:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:3.0:1787829865.318793:0:40701:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:3.0:1787829865.318795:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829865.318797:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787829865.318800:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829865.318807:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829865.318809:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:3.0:1787829865.318813:0:40701:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829865.318823:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787829865.318824:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829865.318827:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787829865.318829:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787829865.318832:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829865.318842:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829865.318849:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c7016d00 x1874670646872448/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787829865.318857:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787829865.318860:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829865.318877:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787829865.318879:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829865.318881:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1787829865.318883:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787829865.318884:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787829865.318886:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787829865.318887:0:40701:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4665 00000100:00080000:3.0:1787829865.318890:0:40701:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787829865.318891:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829865.318892:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:3.0:1787829865.318894:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1787829865.318894:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1787829865.318896:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000020:00000001:3.0:1787829865.318897:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:3.0:1787829865.318899:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829865.318900:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:3.0:1787829865.318901:0:40701:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:3.0:1787829865.318903:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829865.318904:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787829865.318908:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829865.318910:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829865.318911:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:3.0:1787829865.318914:0:40701:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1787829865.318916:0:40701:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829865.318917:0:40701:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829865.318920:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787829865.318921:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829865.318922:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787829865.318923:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787829865.318924:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829865.318929:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829865.318933:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c70170c0 x1874670646872576/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787829865.318939:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787829865.318941:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1787829865.318941:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829865.318947:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829865.318948:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00080000:3.0:1787829865.318949:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:2.0:1787829865.318951:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00080000:3.0:1787829865.318953:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1787829865.318955:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787829865.318956:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:2.0:1787829865.318959:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787829865.318959:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787829865.318965:0:40701:0:(import.c:485:import_select_connection()) Process entered 02000000:00000001:2.0:1787829865.318967:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000100:00080000:3.0:1787829865.318967:0:40701:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4665 02000000:00000001:2.0:1787829865.318972:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00080000:3.0:1787829865.318973:0:40701:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787829865.318976:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829865.318979:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 02000000:00000001:2.0:1787829865.318980:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829865.318981:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1787829865.318983:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1787829865.318986:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00100000:2.0:1787829865.318990:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c7016d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646872448:192.168.203.143@tcp:250:kworker.0 00000020:00000001:3.0:1787829865.318990:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:3.0:1787829865.318992:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829865.318994:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:2.0:1787829865.318997:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00080000:3.0:1787829865.319000:0:40701:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 02000000:00000001:2.0:1787829865.319003:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:3.0:1787829865.319003:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829865.319007:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829865.319007:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787829865.319011:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787829865.319012:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1787829865.319015:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829865.319017:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829865.319017:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:3.0:1787829865.319019:0:40701:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1787829865.319022:0:40701:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829865.319023:0:40701:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829865.319027:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787829865.319027:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829865.319028:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787829865.319029:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787829865.319030:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829865.319032:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787829865.319035:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369547008 : -126234340004608 : ffff8d30c7016d00) 00000100:00000001:3.0:1787829865.319035:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829865.319039:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00100000:3.0:1787829865.319042:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c7017840 x1874670646872704/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787829865.319048:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787829865.319049:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829865.319052:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787829865.319054:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829865.319057:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1787829865.319058:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787829865.319061:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787829865.319064:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787829865.319067:0:40701:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4665 00000100:00080000:3.0:1787829865.319072:0:40701:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000800:00000100:2.0:1787829865.319075:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:3.0:1787829865.319077:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829865.319091:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:3.0:1787829865.319093:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1787829865.319095:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1787829865.319097:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000020:00000001:3.0:1787829865.319100:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:2.0:1787829865.319103:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787829865.319103:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829865.319105:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:2.0:1787829865.319108:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787829865.319111:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787829865.319113:0:40701:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787829865.319114:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1787829865.319121:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829865.319121:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829865.319127:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:2.0:1787829865.319128:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4299339520 02000000:00000001:3.0:1787829865.319130:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829865.319135:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829865.319137:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 00000100:00000001:3.0:1787829865.319138:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:3.0:1787829865.319143:0:40701:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1787829865.319146:0:40701:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829865.319148:0:40701:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829865.319152:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4299339520 02000000:00000001:3.0:1787829865.319154:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:2.0:1787829865.319158:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 02000000:00000001:3.0:1787829865.319158:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829865.319163:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787829865.319167:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787829865.319169:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829865.319171:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787829865.319174:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829865.319177:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829865.319179:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1787829865.319187:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c7014000 x1874670646872832/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1787829865.319188:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829865/real 1787829865] req@ffff8d30c7016d00 x1874670646872448/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829890 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829865.319213:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787829865.319217:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:2.0:1787829865.319220:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000001:3.0:1787829865.319221:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:2.0:1787829865.319223:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1787829865.319227:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1787829865.319230:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787829865.319231:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829865.319233:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787829865.319235:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787829865.319239:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c7016d00 x1874670646872448/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829890 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787829865.319245:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829865.319248:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787829865.319251:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829865.319252:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787829865.319253:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829865.319254:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787829865.319255:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787829865.319257:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829865.319261:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:2.0:1787829865.319263:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1787829865.319265:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c7016d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646872448:192.168.203.143@tcp:250:kworker.0 00000100:00000001:2.0:1787829865.319270:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829865.319272:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787829865.319274:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787829865.319276:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1787829865.319279:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787829865.319281:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:2.0:1787829865.319283:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:2.0:1787829865.319287:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787829865.319288:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1787829865.319293:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787829865.319294:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829865.319296:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829865.319302:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1787829865.319303:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829865.319305:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787829865.319306:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1787829865.319307:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1787829865.319308:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787829865.319309:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829865.319311:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787829865.319313:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787829865.319314:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787829865.319317:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c70170c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646872576:192.168.203.143@tcp:38:kworker.0 00000100:00000001:2.0:1787829865.319320:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787829865.319321:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1787829865.319323:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829865.319324:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1787829865.319327:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829865.319332:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787829865.319333:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369547968 : -126234340003648 : ffff8d30c70170c0) 00000100:00000001:2.0:1787829865.319337:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1787829865.319357:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:2.0:1787829865.319361:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1787829865.319362:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787829865.319364:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829865.319365:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1787829865.319368:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829865.319371:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4299338262 00000100:00000001:2.0:1787829865.319373:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4299339520 00000100:00000001:2.0:1787829865.319375:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4299338262 00000100:00000001:2.0:1787829865.319377:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4299339520 00000100:00000001:2.0:1787829865.319378:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787829865.319379:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829865.319381:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1787829865.319385:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829865/real 1787829865] req@ffff8d30c70170c0 x1874670646872576/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829890 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1787829865.319393:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787829865.319394:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1787829865.319397:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1787829865.319398:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787829865.319400:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829865.319401:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787829865.319402:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787829865.319406:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c70170c0 x1874670646872576/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829890 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787829865.319411:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829865.319413:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787829865.319415:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829865.319417:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787829865.319418:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829865.319419:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787829865.319421:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787829865.319424:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829865.319427:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:2.0:1787829865.319430:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1787829865.319439:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c70170c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646872576:192.168.203.143@tcp:38:kworker.0 00000100:00000001:2.0:1787829865.319443:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1787829865.319445:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787829865.319446:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829865.319448:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787829865.319450:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787829865.319452:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787829865.319454:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c7017840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646872704:192.168.203.143@tcp:38:kworker.0 00000100:00000001:2.0:1787829865.319458:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787829865.319461:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1787829865.319463:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829865.319464:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1787829865.319467:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829865.319472:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787829865.319474:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369549888 : -126234340001728 : ffff8d30c7017840) 00000100:00000001:2.0:1787829865.319477:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1787829865.319487:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:2.0:1787829865.319491:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1787829865.319495:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787829865.319496:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829865.319498:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1787829865.319501:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829865.319504:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4299338262 00000100:00000001:2.0:1787829865.319506:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4299339520 00000100:00000001:2.0:1787829865.319508:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4299338262 00000100:00000001:2.0:1787829865.319509:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4299339520 00000100:00000001:2.0:1787829865.319511:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787829865.319513:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829865.319514:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1787829865.319518:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829865/real 1787829865] req@ffff8d30c7017840 x1874670646872704/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829890 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1787829865.319529:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787829865.319530:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1787829865.319532:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1787829865.319533:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787829865.319534:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829865.319535:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787829865.319536:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787829865.319539:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c7017840 x1874670646872704/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829890 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787829865.319543:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829865.319545:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787829865.319547:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829865.319548:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787829865.319548:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829865.319550:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787829865.319551:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787829865.319552:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829865.319554:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:2.0:1787829865.319556:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1787829865.319557:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c7017840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646872704:192.168.203.143@tcp:38:kworker.0 00000100:00000001:2.0:1787829865.319560:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1787829865.319561:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787829865.319561:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829865.319563:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787829865.319564:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787829865.319565:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787829865.319567:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c7014000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646872832:192.168.203.143@tcp:8:kworker.0 00000100:00000001:2.0:1787829865.319569:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787829865.319570:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1787829865.319571:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829865.319572:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1787829865.319574:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829865.319578:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787829865.319579:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369535488 : -126234340016128 : ffff8d30c7014000) 00000100:00000001:2.0:1787829865.319581:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1787829865.319593:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:2.0:1787829865.319597:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1787829865.319599:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787829865.319621:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829865.319622:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1787829865.319626:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829865.319628:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4299338276 00000100:00000001:2.0:1787829865.319630:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4299339520 00000100:00000001:2.0:1787829865.319632:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4299338276 00000100:00000001:2.0:1787829865.319634:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4299339520 00000100:00000001:2.0:1787829865.319636:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787829865.319637:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829865.319639:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1787829865.319643:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829865/real 1787829865] req@ffff8d30c7014000 x1874670646872832/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829890 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1787829865.319652:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787829865.319653:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1787829865.319657:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1787829865.319659:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787829865.319661:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829865.319663:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787829865.319665:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787829865.319670:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c7014000 x1874670646872832/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829890 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787829865.319678:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829865.319680:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787829865.319683:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829865.319686:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787829865.319687:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829865.319689:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787829865.319691:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787829865.319694:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829865.319699:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:2.0:1787829865.319702:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1787829865.319705:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c7014000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646872832:192.168.203.143@tcp:8:kworker.0 00000100:00000001:2.0:1787829865.319709:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829865.319711:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787829865.319712:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787829865.319714:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1787829865.319716:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787829865.319717:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:2.0:1787829865.319718:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:2.0:1787829865.319720:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787829865.319721:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1787829865.319723:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787829865.319724:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829865.319725:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787829865.319727:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787829865.319728:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1787829865.319729:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787829865.319730:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:2.0:1787829865.319731:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:2.0:1787829865.319732:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787829865.319733:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1787829865.319734:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787829865.319735:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829865.319736:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787829865.319737:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787829865.319738:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1787829865.319739:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787829865.319740:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:2.0:1787829865.319741:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:2.0:1787829865.319742:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787829865.319743:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1787829865.319744:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787829865.319745:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829865.319746:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829865.319749:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1787829865.319750:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829865.319751:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787829865.319753:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829866.214686:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829866.214693:0:815:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1787829866.214695:0:815:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1787829866.214698:0:815:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829866.214702:0:815:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1787829866.214704:0:815:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829866.214706:0:815:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829866.214708:0:815:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829866.214762:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829866.214767:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829866.214771:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829867.238736:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829867.238744:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829867.238842:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829867.238847:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829867.238852:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829868.262674:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829868.262682:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829868.262741:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829868.262745:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829868.262747:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829869.286659:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829869.286663:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829869.286711:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829869.286714:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829869.286717:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829870.310667:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829870.310673:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829870.310719:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829870.310722:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829870.310726:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1787829870.438674:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829870.438682:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:2.0:1787829870.438685:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787829870.438688:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787829870.438692:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787829870.438697:0:40798:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 4670 00000100:00080000:2.0:1787829870.438706:0:40798:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787829870.438708:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829870.438709:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787829870.438712:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787829870.438714:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787829870.438716:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:2.0:1787829870.438718:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:2.0:1787829870.438719:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829870.438720:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:2.0:1787829870.438724:0:40798:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787829870.438726:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829870.438727:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787829870.438730:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787829870.438732:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829870.438733:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:2.0:1787829870.438735:0:40798:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829870.438745:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787829870.438746:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829870.438749:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787829870.438752:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787829870.438756:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829870.438775:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787829870.438782:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2ed00 x1874670646872960/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787829870.438790:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787829870.438793:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829870.438807:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787829870.438809:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829870.438811:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:2.0:1787829870.438815:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787829870.438817:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787829870.438819:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787829870.438820:0:40798:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4670 00000100:00080000:2.0:1787829870.438823:0:40798:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787829870.438824:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829870.438826:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787829870.438827:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787829870.438829:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:2.0:1787829870.438830:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000020:00000001:2.0:1787829870.438832:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:2.0:1787829870.438834:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829870.438835:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:2.0:1787829870.438838:0:40798:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787829870.438840:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829870.438842:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:2.0:1787829870.438842:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:0.0:1787829870.438847:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787829870.438850:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 02000000:00000001:2.0:1787829870.438851:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787829870.438852:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787829870.438855:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829870.438857:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829870.438859:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:2.0:1787829870.438861:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 02000000:00000001:0.0:1787829870.438863:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787829870.438866:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000008:00100000:2.0:1787829870.438868:0:40798:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829870.438871:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646872960:192.168.203.143@tcp:250:kworker.0 00000008:00000001:2.0:1787829870.438874:0:40798:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829870.438876:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:2.0:1787829870.438880:0:40798:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829870.438881:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787829870.438885:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829870.438888:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1787829870.438888:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787829870.438892:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829870.438893:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829870.438897:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787829870.438900:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787829870.438903:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829870.438909:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787829870.438909:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829870.438912:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417879808 : -126234291671808 : ffff8d30c9e2ed00) 00000100:00000001:0.0:1787829870.438915:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00100000:2.0:1787829870.438918:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2f0c0 x1874670646873088/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787829870.438925:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787829870.438927:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829870.438930:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787829870.438932:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1787829870.438935:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1787829870.438936:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:2.0:1787829870.438937:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787829870.438939:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:2.0:1787829870.438941:0:40798:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4670 00000100:00080000:2.0:1787829870.438944:0:40798:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787829870.438946:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829870.438946:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787829870.438948:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:2.0:1787829870.438949:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000800:00000100:0.0:1787829870.438953:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000020:00000001:2.0:1787829870.438953:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000020:00000001:2.0:1787829870.438957:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:2.0:1787829870.438959:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829870.438960:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:0.0:1787829870.438964:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00080000:2.0:1787829870.438965:0:40798:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787829870.438970:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787829870.438970:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829870.438974:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829870.438975:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:0.0:1787829870.438979:0:2390:0:(events.c:68:request_out_callback()) Process leaving 02000000:00000001:2.0:1787829870.438979:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787829870.438983:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829870.438984:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829870.438985:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:2.0:1787829870.438988:0:40798:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00000001:0.0:1787829870.438989:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4299344640 00000008:00000001:2.0:1787829870.438992:0:40798:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829870.438994:0:40798:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829870.438998:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 02000000:00000001:2.0:1787829870.438999:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787829870.439002:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829870.439004:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787829870.439005:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787829870.439006:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829870.439007:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4299344640 00000100:00000001:2.0:1787829870.439010:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829870.439012:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 00000100:00100000:2.0:1787829870.439016:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2c780 x1874670646873216/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787829870.439018:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787829870.439022:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829870.439023:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787829870.439028:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829870/real 1787829870] req@ffff8d30c9e2ed00 x1874670646872960/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829895 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787829870.439056:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787829870.439060:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:0.0:1787829870.439063:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000001:2.0:1787829870.439063:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:0.0:1787829870.439066:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00080000:2.0:1787829870.439068:0:40798:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1787829870.439070:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00080000:2.0:1787829870.439071:0:40798:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1787829870.439074:0:40798:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:0.0:1787829870.439075:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00080000:2.0:1787829870.439076:0:40798:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787829870.439078:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829870.439080:0:40798:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:0.0:1787829870.439081:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787829870.439084:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:2.0:1787829870.439087:0:40798:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4670 00000100:00100000:0.0:1787829870.439091:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2ed00 x1874670646872960/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829895 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1787829870.439091:0:40798:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787829870.439100:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829870.439100:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829870.439104:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:0.0:1787829870.439105:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000020:00000001:2.0:1787829870.439108:0:40798:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:0.0:1787829870.439109:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:2.0:1787829870.439111:0:40798:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:0.0:1787829870.439112:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787829870.439114:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:2.0:1787829870.439114:0:40798:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:0.0:1787829870.439119:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000020:00000001:2.0:1787829870.439120:0:40798:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:0.0:1787829870.439121:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787829870.439122:0:40798:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00080000:0.0:1787829870.439126:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00000001:2.0:1787829870.439128:0:40798:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:0.0:1787829870.439131:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787829870.439135:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00080000:2.0:1787829870.439135:0:40798:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00100000:0.0:1787829870.439141:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2ed00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646872960:192.168.203.143@tcp:250:kworker.0 00000100:00000001:2.0:1787829870.439142:0:40798:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829870.439147:0:40798:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:0.0:1787829870.439149:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:2.0:1787829870.439150:0:40798:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787829870.439152:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 02000000:00000001:2.0:1787829870.439154:0:40798:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829870.439155:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 00000100:00000001:2.0:1787829870.439158:0:40798:0:(obd_class.h:888:obd_reconnect()) Process entered 02000000:00000001:0.0:1787829870.439159:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 00000008:00100000:2.0:1787829870.439161:0:40798:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 02000000:00000001:0.0:1787829870.439162:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000008:00000001:2.0:1787829870.439164:0:40798:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787829870.439166:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000100:00000001:2.0:1787829870.439167:0:40798:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787829870.439170:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:2.0:1787829870.439173:0:40798:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787829870.439175:0:40798:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829870.439176:0:40798:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787829870.439177:0:40798:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787829870.439178:0:40798:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829870.439180:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 00000100:00000001:2.0:1787829870.439182:0:40798:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829870.439183:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787829870.439190:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00100000:2.0:1787829870.439191:0:40798:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c9e2fc00 x1874670646873344/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787829870.439192:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829870.439197:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829870.439203:0:40798:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787829870.439204:0:40798:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829870.439205:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1787829870.439206:0:40798:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829870.439209:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829870.439212:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787829870.439213:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787829870.439215:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1787829870.439216:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787829870.439217:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829870.439219:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787829870.439221:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787829870.439222:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787829870.439226:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646873088:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787829870.439228:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787829870.439230:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787829870.439231:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829870.439233:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1787829870.439235:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829870.439239:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787829870.439240:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417880768 : -126234291670848 : ffff8d30c9e2f0c0) 00000100:00000001:0.0:1787829870.439243:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1787829870.439253:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:0.0:1787829870.439256:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1787829870.439257:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787829870.439258:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829870.439259:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787829870.439261:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829870.439265:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4299343382 00000100:00000001:0.0:1787829870.439267:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4299344640 00000100:00000001:0.0:1787829870.439268:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4299343382 00000100:00000001:0.0:1787829870.439270:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4299344640 00000100:00000001:0.0:1787829870.439272:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787829870.439273:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829870.439274:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787829870.439277:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829870/real 1787829870] req@ffff8d30c9e2f0c0 x1874670646873088/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829895 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787829870.439283:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787829870.439284:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787829870.439285:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787829870.439286:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787829870.439287:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829870.439288:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787829870.439289:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787829870.439291:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2f0c0 x1874670646873088/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829895 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787829870.439296:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829870.439297:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787829870.439299:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829870.439300:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787829870.439300:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829870.439301:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787829870.439303:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787829870.439304:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829870.439306:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787829870.439307:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787829870.439309:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2f0c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646873088:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787829870.439311:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1787829870.439312:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787829870.439313:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829870.439314:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787829870.439315:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787829870.439316:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787829870.439318:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646873216:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787829870.439321:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787829870.439322:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787829870.439323:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829870.439324:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1787829870.439326:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829870.439328:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787829870.439329:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417870208 : -126234291681408 : ffff8d30c9e2c780) 00000100:00000001:0.0:1787829870.439331:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1787829870.439336:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:0.0:1787829870.439339:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1787829870.439340:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787829870.439341:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829870.439342:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787829870.439343:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829870.439345:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4299343382 00000100:00000001:0.0:1787829870.439347:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4299344640 00000100:00000001:0.0:1787829870.439348:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4299343382 00000100:00000001:0.0:1787829870.439350:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4299344640 00000100:00000001:0.0:1787829870.439351:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787829870.439352:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829870.439353:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787829870.439355:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829870/real 1787829870] req@ffff8d30c9e2c780 x1874670646873216/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829895 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787829870.439360:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787829870.439361:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787829870.439362:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787829870.439363:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787829870.439363:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829870.439364:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787829870.439365:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787829870.439367:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2c780 x1874670646873216/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829895 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787829870.439376:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829870.439377:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787829870.439379:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829870.439380:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787829870.439381:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829870.439381:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787829870.439382:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787829870.439384:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829870.439385:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787829870.439387:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787829870.439388:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2c780 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646873216:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787829870.439390:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1787829870.439391:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787829870.439392:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829870.439393:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787829870.439394:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787829870.439395:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787829870.439397:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c9e2fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646873344:192.168.203.143@tcp:8:kworker.0 00000100:00000001:0.0:1787829870.439399:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787829870.439400:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787829870.439400:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829870.439401:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1787829870.439403:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829870.439406:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787829870.439407:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839417883648 : -126234291667968 : ffff8d30c9e2fc00) 00000100:00000001:0.0:1787829870.439409:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1787829870.439414:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:0.0:1787829870.439416:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1787829870.439418:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787829870.439418:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829870.439420:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787829870.439421:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829870.439423:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4299343395 00000100:00000001:0.0:1787829870.439425:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4299344640 00000100:00000001:0.0:1787829870.439427:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4299343395 00000100:00000001:0.0:1787829870.439428:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4299344640 00000100:00000001:0.0:1787829870.439429:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787829870.439430:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829870.439431:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787829870.439434:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829870/real 1787829870] req@ffff8d30c9e2fc00 x1874670646873344/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829895 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787829870.439438:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787829870.439439:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787829870.439440:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787829870.439441:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787829870.439442:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829870.439443:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787829870.439444:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787829870.439448:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c9e2fc00 x1874670646873344/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829895 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787829870.439455:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829870.439457:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787829870.439460:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829870.439461:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787829870.439463:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829870.439464:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787829870.439466:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787829870.439469:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829870.439472:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787829870.439474:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787829870.439478:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c9e2fc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646873344:192.168.203.143@tcp:8:kworker.0 00000100:00000001:0.0:1787829870.439482:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829870.439484:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787829870.439485:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787829870.439487:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787829870.439489:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787829870.439491:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787829870.439492:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787829870.439494:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787829870.439496:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787829870.439498:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787829870.439499:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829870.439501:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787829870.439502:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787829870.439504:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787829870.439505:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787829870.439507:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787829870.439508:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787829870.439510:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787829870.439510:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787829870.439512:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787829870.439513:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829870.439514:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787829870.439515:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787829870.439516:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787829870.439517:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787829870.439517:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787829870.439518:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787829870.439519:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787829870.439520:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787829870.439520:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787829870.439521:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829870.439522:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829870.439524:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787829870.439525:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829870.439526:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787829870.439527:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829871.334689:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829871.334696:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829871.334743:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829871.334747:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829871.334751:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829872.358754:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829872.358763:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829872.358830:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829872.358839:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829872.358843:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829873.382676:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829873.382681:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829873.382721:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829873.382725:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829873.382727:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829874.406713:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829874.406719:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829874.406851:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829874.406855:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829874.406858:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829875.430700:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829875.430706:0:815:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1787829875.430708:0:815:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1787829875.430710:0:815:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829875.430714:0:815:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1787829875.430716:0:815:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829875.430717:0:815:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829875.430719:0:815:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829875.430773:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829875.430777:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829875.430780:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787829875.558713:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829875.558721:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1787829875.558724:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787829875.558734:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787829875.558738:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787829875.558744:0:40701:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 4676 00000100:00080000:3.0:1787829875.558753:0:40701:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787829875.558755:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829875.558757:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:3.0:1787829875.558760:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1787829875.558762:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1787829875.558764:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:3.0:1787829875.558766:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:3.0:1787829875.558767:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829875.558768:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:3.0:1787829875.558772:0:40701:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:3.0:1787829875.558774:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829875.558776:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787829875.558779:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829875.558781:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829875.558783:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:3.0:1787829875.558784:0:40701:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829875.558794:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787829875.558796:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829875.558798:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787829875.558800:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787829875.558803:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829875.558812:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829875.558820:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c7017c00 x1874670646873472/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787829875.558828:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787829875.558831:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829875.558842:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787829875.558844:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829875.558847:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1787829875.558848:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787829875.558849:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787829875.558851:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787829875.558852:0:40701:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4676 00000100:00080000:3.0:1787829875.558854:0:40701:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787829875.558855:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829875.558856:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:3.0:1787829875.558858:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1787829875.558858:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1787829875.558860:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000020:00000001:3.0:1787829875.558862:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:3.0:1787829875.558863:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829875.558865:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:3.0:1787829875.558867:0:40701:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:3.0:1787829875.558869:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829875.558871:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787829875.558875:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829875.558878:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829875.558879:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1787829875.558883:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000008:00100000:3.0:1787829875.558883:0:40701:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1787829875.558888:0:40701:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829875.558889:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787829875.558890:0:40701:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829875.558894:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 02000000:00000001:3.0:1787829875.558895:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787829875.558899:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829875.558900:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787829875.558900:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787829875.558904:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:0.0:1787829875.558905:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829875.558906:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829875.558914:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:3.0:1787829875.558914:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829875.558919:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00100000:3.0:1787829875.558920:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c7016940 x1874670646873600/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787829875.558929:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787829875.558931:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829875.558933:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829875.558934:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787829875.558935:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829875.558939:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1787829875.558940:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787829875.558942:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787829875.558946:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00100000:0.0:1787829875.558948:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c7017c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646873472:192.168.203.143@tcp:250:kworker.0 00000100:00080000:3.0:1787829875.558950:0:40701:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4676 00000100:00080000:3.0:1787829875.558956:0:40701:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787829875.558957:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:3.0:1787829875.558959:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829875.558961:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:3.0:1787829875.558962:0:40701:0:(genops.c:792:class_conn2export()) Process entered 02000000:00000001:0.0:1787829875.558964:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000020:00000001:3.0:1787829875.558964:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1787829875.558968:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 02000000:00000001:0.0:1787829875.558969:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829875.558970:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 02000000:00000001:0.0:1787829875.558975:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:3.0:1787829875.558975:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829875.558976:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:3.0:1787829875.558980:0:40701:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:3.0:1787829875.558982:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829875.558984:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829875.558984:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787829875.558987:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829875.558990:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829875.558991:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:3.0:1787829875.558992:0:40701:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1787829875.558994:0:40701:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829875.558995:0:40701:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829875.558999:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787829875.558999:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829875.559001:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:0.0:1787829875.559002:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 02000000:00000001:3.0:1787829875.559002:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:0.0:1787829875.559006:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369550848 : -126234340000768 : ffff8d30c7017c00) 02000000:00000001:3.0:1787829875.559007:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829875.559026:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:3.0:1787829875.559028:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829875.559033:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c7014000 x1874670646873728/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787829875.559039:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787829875.559040:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829875.559042:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787829875.559045:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829875.559047:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1787829875.559048:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787829875.559049:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787829875.559051:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787829875.559053:0:40701:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4676 00000100:00080000:3.0:1787829875.559056:0:40701:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787829875.559058:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829875.559059:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:3.0:1787829875.559060:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1787829875.559061:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1787829875.559063:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000020:00000001:3.0:1787829875.559066:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000800:00000100:0.0:1787829875.559068:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:3.0:1787829875.559069:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829875.559073:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:3.0:1787829875.559076:0:40701:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:3.0:1787829875.559079:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829875.559080:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:0.0:1787829875.559083:0:2390:0:(events.c:34:request_out_callback()) Process entered 02000000:00000001:3.0:1787829875.559083:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829875.559088:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829875.559090:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829875.559090:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:3.0:1787829875.559091:0:40701:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00000001:0.0:1787829875.559095:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:3.0:1787829875.559096:0:40701:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829875.559102:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1787829875.559102:0:40701:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829875.559107:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787829875.559108:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829875.559109:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787829875.559110:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787829875.559111:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829875.559114:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829875.559118:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c7017840 x1874670646873856/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787829875.559122:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829875.559124:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787829875.559128:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829875.559129:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4299349760 00000100:00000001:3.0:1787829875.559130:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829875.559143:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1787829875.559152:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4299349760 00000100:00000001:0.0:1787829875.559157:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1787829875.559161:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787829875.559164:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829875.559166:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787829875.559173:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829875/real 1787829875] req@ffff8d30c7017c00 x1874670646873472/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829900 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787829875.559184:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787829875.559186:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787829875.559189:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787829875.559192:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787829875.559194:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829875.559197:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787829875.559199:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787829875.559204:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c7017c00 x1874670646873472/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829900 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787829875.559212:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829875.559216:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787829875.559218:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829875.559220:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787829875.559221:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829875.559224:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787829875.559225:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787829875.559227:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829875.559231:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787829875.559232:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787829875.559235:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c7017c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646873472:192.168.203.143@tcp:250:kworker.0 00000100:00000001:0.0:1787829875.559239:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829875.559241:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787829875.559242:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787829875.559244:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787829875.559246:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787829875.559247:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787829875.559249:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787829875.559252:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787829875.559253:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787829875.559255:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787829875.559256:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829875.559258:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829875.559264:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787829875.559265:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829875.559266:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787829875.559267:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787829875.559269:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1787829875.559270:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787829875.559271:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829875.559272:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787829875.559274:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787829875.559276:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787829875.559278:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c7016940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646873600:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787829875.559280:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787829875.559281:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787829875.559282:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829875.559283:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1787829875.559286:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829875.559290:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787829875.559291:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369546048 : -126234340005568 : ffff8d30c7016940) 00000100:00000001:0.0:1787829875.559295:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1787829875.559304:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:0.0:1787829875.559308:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1787829875.559309:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787829875.559310:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829875.559312:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787829875.559314:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829875.559317:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4299348502 00000100:00000001:0.0:1787829875.559319:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4299349760 00000100:00000001:0.0:1787829875.559321:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4299348502 00000100:00000001:0.0:1787829875.559322:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4299349760 00000100:00000001:0.0:1787829875.559324:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787829875.559325:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829875.559326:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787829875.559329:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829875/real 1787829875] req@ffff8d30c7016940 x1874670646873600/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829900 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787829875.559334:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787829875.559335:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787829875.559336:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787829875.559337:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787829875.559338:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829875.559339:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787829875.559340:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787829875.559343:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c7016940 x1874670646873600/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829900 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787829875.559347:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829875.559349:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787829875.559350:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829875.559351:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787829875.559352:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829875.559353:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787829875.559354:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787829875.559356:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829875.559358:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787829875.559359:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787829875.559361:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c7016940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646873600:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787829875.559363:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1787829875.559364:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787829875.559365:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829875.559366:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787829875.559367:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787829875.559368:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787829875.559370:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c7014000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646873728:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787829875.559372:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787829875.559373:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787829875.559374:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829875.559375:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1787829875.559377:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829875.559380:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787829875.559381:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369535488 : -126234340016128 : ffff8d30c7014000) 00000100:00000001:0.0:1787829875.559383:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1787829875.559389:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:0.0:1787829875.559391:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1787829875.559392:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787829875.559394:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829875.559395:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787829875.559396:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829875.559398:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4299348502 00000100:00000001:0.0:1787829875.559400:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4299349760 00000100:00000001:0.0:1787829875.559402:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4299348502 00000100:00000001:0.0:1787829875.559413:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4299349760 00000100:00000001:0.0:1787829875.559414:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787829875.559415:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829875.559416:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787829875.559419:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829875/real 1787829875] req@ffff8d30c7014000 x1874670646873728/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829900 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787829875.559425:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787829875.559426:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787829875.559427:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787829875.559428:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787829875.559429:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829875.559430:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787829875.559431:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787829875.559433:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c7014000 x1874670646873728/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829900 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787829875.559438:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829875.559439:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787829875.559440:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829875.559441:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787829875.559442:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829875.559443:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787829875.559444:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787829875.559445:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829875.559447:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787829875.559449:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787829875.559451:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c7014000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646873728:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787829875.559454:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1787829875.559454:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787829875.559456:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829875.559457:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787829875.559458:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787829875.559459:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787829875.559461:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c7017840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646873856:192.168.203.143@tcp:8:kworker.0 00000100:00000001:0.0:1787829875.559463:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787829875.559464:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787829875.559465:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829875.559466:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1787829875.559468:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829875.559472:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787829875.559472:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369549888 : -126234340001728 : ffff8d30c7017840) 00000100:00000001:0.0:1787829875.559475:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1787829875.559481:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:0.0:1787829875.559483:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1787829875.559485:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787829875.559486:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829875.559487:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787829875.559488:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829875.559491:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4299348515 00000100:00000001:0.0:1787829875.559492:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4299349760 00000100:00000001:0.0:1787829875.559494:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4299348515 00000100:00000001:0.0:1787829875.559495:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4299349760 00000100:00000001:0.0:1787829875.559497:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787829875.559498:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829875.559499:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787829875.559501:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829875/real 1787829875] req@ffff8d30c7017840 x1874670646873856/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829900 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787829875.559511:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787829875.559512:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787829875.559513:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787829875.559514:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787829875.559515:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829875.559516:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787829875.559516:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787829875.559519:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c7017840 x1874670646873856/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829900 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787829875.559523:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829875.559524:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787829875.559525:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829875.559526:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787829875.559527:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829875.559528:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787829875.559529:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787829875.559530:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829875.559532:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787829875.559533:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787829875.559535:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c7017840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646873856:192.168.203.143@tcp:8:kworker.0 00000100:00000001:0.0:1787829875.559537:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829875.559538:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787829875.559539:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787829875.559540:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787829875.559542:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787829875.559543:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787829875.559544:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787829875.559545:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787829875.559546:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787829875.559547:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787829875.559548:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829875.559549:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787829875.559550:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787829875.559551:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787829875.559552:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787829875.559553:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787829875.559554:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787829875.559555:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787829875.559556:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787829875.559557:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787829875.559557:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829875.559558:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787829875.559559:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787829875.559560:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787829875.559560:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787829875.559561:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787829875.559562:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787829875.559563:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787829875.559563:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787829875.559564:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787829875.559565:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829875.559566:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829875.559567:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787829875.559568:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829875.559569:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787829875.559570:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829876.454735:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829876.454741:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829876.454812:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829876.454816:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829876.454820:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829877.478731:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829877.478739:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829877.478806:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829877.478811:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829877.478814:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829878.502710:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829878.502717:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829878.502801:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829878.502808:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829878.502813:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829879.526722:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829879.526727:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829879.526811:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829879.526816:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829879.526820:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829880.550700:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829880.550706:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829880.550784:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829880.550789:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829880.550794:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787829880.678741:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829880.678760:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1787829880.678763:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787829880.678768:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787829880.678774:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787829880.678784:0:40701:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 4681 00000100:00080000:3.0:1787829880.678797:0:40701:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787829880.678801:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829880.678804:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:3.0:1787829880.678809:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1787829880.678813:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1787829880.678816:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:3.0:1787829880.678819:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:3.0:1787829880.678821:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829880.678823:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:3.0:1787829880.678828:0:40701:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:3.0:1787829880.678832:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829880.678835:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787829880.678840:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829880.678843:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829880.678846:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:3.0:1787829880.678850:0:40701:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829880.678865:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787829880.678868:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829880.678871:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787829880.678875:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787829880.678880:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829880.678895:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829880.678906:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c70170c0 x1874670646873984/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787829880.678918:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787829880.678924:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829880.678945:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787829880.678949:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829880.678953:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1787829880.678955:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787829880.678957:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787829880.678960:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787829880.678963:0:40701:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4681 00000100:00080000:3.0:1787829880.678968:0:40701:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787829880.678971:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829880.678986:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:3.0:1787829880.678989:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:0.0:1787829880.678991:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:3.0:1787829880.678991:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1787829880.678997:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:0.0:1787829880.678998:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000020:00000001:3.0:1787829880.679000:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:0.0:1787829880.679002:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787829880.679004:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829880.679006:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787829880.679008:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:0.0:1787829880.679010:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787829880.679016:0:40701:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:3.0:1787829880.679020:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829880.679022:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787829880.679023:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787829880.679026:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787829880.679030:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829880.679032:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00100000:0.0:1787829880.679036:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c70170c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646873984:192.168.203.143@tcp:250:kworker.0 02000000:00000001:3.0:1787829880.679038:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829880.679044:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1787829880.679045:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000008:00100000:3.0:1787829880.679050:0:40701:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 02000000:00000001:0.0:1787829880.679052:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787829880.679056:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:3.0:1787829880.679057:0:40701:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829880.679059:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:3.0:1787829880.679062:0:40701:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829880.679065:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829880.679069:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787829880.679072:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829880.679074:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787829880.679076:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787829880.679078:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829880.679085:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829880.679090:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00100000:3.0:1787829880.679092:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c7016d00 x1874670646874112/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787829880.679093:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369547968 : -126234340003648 : ffff8d30c70170c0) 00000100:00000001:0.0:1787829880.679100:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:3.0:1787829880.679123:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787829880.679125:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829880.679129:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787829880.679132:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829880.679136:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1787829880.679138:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787829880.679140:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:0.0:1787829880.679144:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:3.0:1787829880.679145:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787829880.679151:0:40701:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4681 00000100:00080000:3.0:1787829880.679156:0:40701:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787829880.679159:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787829880.679159:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829880.679163:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:0.0:1787829880.679166:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000020:00000001:3.0:1787829880.679168:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:0.0:1787829880.679169:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829880.679172:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000020:00000001:3.0:1787829880.679172:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1787829880.679177:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:0.0:1787829880.679180:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829880.679181:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:0.0:1787829880.679185:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4299354880 00000100:00000001:3.0:1787829880.679186:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829880.679191:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:3.0:1787829880.679195:0:40701:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787829880.679198:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 00000100:00000001:3.0:1787829880.679199:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829880.679203:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:0.0:1787829880.679208:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4299354880 02000000:00000001:3.0:1787829880.679208:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829880.679214:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829880.679216:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 00000100:00000001:3.0:1787829880.679218:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1787829880.679220:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000008:00100000:3.0:1787829880.679222:0:40701:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00000001:0.0:1787829880.679223:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:3.0:1787829880.679228:0:40701:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829880.679229:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00000001:3.0:1787829880.679233:0:40701:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787829880.679237:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829880/real 1787829880] req@ffff8d30c70170c0 x1874670646873984/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829905 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 02000000:00000001:3.0:1787829880.679240:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787829880.679244:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829880.679249:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000400:00000001:0.0:1787829880.679252:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 02000000:00000001:3.0:1787829880.679253:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000400:00000001:0.0:1787829880.679255:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 02000000:00000001:3.0:1787829880.679258:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829880.679259:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787829880.679263:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787829880.679265:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829880.679266:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829880.679267:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787829880.679270:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829880.679276:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c70143c0 x1874670646874240/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829880.679277:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c70170c0 x1874670646873984/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829905 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787829880.679305:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829880.679309:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787829880.679310:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787829880.679315:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829880.679315:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829880.679321:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787829880.679323:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829880.679323:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829880.679330:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00080000:3.0:1787829880.679330:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1787829880.679332:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787829880.679338:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829880.679340:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1787829880.679343:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787829880.679344:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787829880.679348:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00080000:3.0:1787829880.679348:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1787829880.679355:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c70170c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646873984:192.168.203.143@tcp:250:kworker.0 00000100:00000001:3.0:1787829880.679355:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:0.0:1787829880.679363:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:3.0:1787829880.679363:0:40701:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4681 00000100:00000001:0.0:1787829880.679366:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787829880.679367:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787829880.679369:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787829880.679371:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787829880.679373:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787829880.679376:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 00000100:00080000:3.0:1787829880.679379:0:40701:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 02000000:00000001:0.0:1787829880.679381:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 00000100:00000001:3.0:1787829880.679383:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 02000000:00000001:0.0:1787829880.679384:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829880.679388:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:0.0:1787829880.679389:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787829880.679391:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:3.0:1787829880.679394:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:0.0:1787829880.679395:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:3.0:1787829880.679398:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1787829880.679401:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000020:00000001:3.0:1787829880.679404:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:3.0:1787829880.679407:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829880.679409:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:0.0:1787829880.679411:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00080000:3.0:1787829880.679413:0:40701:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787829880.679415:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829880.679420:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1787829880.679420:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829880.679423:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 02000000:00000001:3.0:1787829880.679425:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:0.0:1787829880.679426:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1787829880.679429:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787829880.679430:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829880.679432:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787829880.679433:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787829880.679436:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829880.679438:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829880.679439:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829880.679443:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00100000:0.0:1787829880.679445:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c7016d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646874112:192.168.203.143@tcp:38:kworker.0 00000008:00100000:3.0:1787829880.679447:0:40701:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00000001:0.0:1787829880.679449:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787829880.679452:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000008:00000001:3.0:1787829880.679454:0:40701:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829880.679455:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829880.679458:0:40701:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829880.679459:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1787829880.679463:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829880.679466:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:0.0:1787829880.679470:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 02000000:00000001:3.0:1787829880.679470:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829880.679474:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369547008 : -126234340004608 : ffff8d30c7016d00) 00000100:00000001:3.0:1787829880.679474:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787829880.679479:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:0.0:1787829880.679482:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 02000000:00000001:3.0:1787829880.679483:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829880.679492:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:0.0:1787829880.679497:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00100000:3.0:1787829880.679500:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c70152c0 x1874670646874368/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787829880.679502:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1787829880.679505:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787829880.679508:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829880.679512:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787829880.679515:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829880.679518:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787829880.679521:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4299353622 00000100:00000001:3.0:1787829880.679522:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829880.679525:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4299354880 00000100:00000001:0.0:1787829880.679528:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4299353622 00000100:00000001:3.0:1787829880.679529:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829880.679531:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4299354880 00000100:00000001:0.0:1787829880.679535:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787829880.679537:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829880.679538:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787829880.679543:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829880/real 1787829880] req@ffff8d30c7016d00 x1874670646874112/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829905 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787829880.679549:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787829880.679550:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787829880.679551:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787829880.679553:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787829880.679554:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829880.679555:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787829880.679557:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787829880.679560:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c7016d00 x1874670646874112/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829905 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787829880.679565:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829880.679567:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787829880.679569:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829880.679570:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787829880.679571:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829880.679572:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787829880.679580:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787829880.679582:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829880.679585:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787829880.679587:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787829880.679590:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c7016d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646874112:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787829880.679593:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1787829880.679594:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787829880.679595:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829880.679596:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787829880.679597:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787829880.679621:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787829880.679624:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c70143c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646874240:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787829880.679627:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787829880.679628:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787829880.679629:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829880.679631:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1787829880.679633:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829880.679637:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787829880.679638:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369536448 : -126234340015168 : ffff8d30c70143c0) 00000100:00000001:0.0:1787829880.679641:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1787829880.679649:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:0.0:1787829880.679653:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1787829880.679654:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787829880.679655:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829880.679657:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787829880.679659:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829880.679661:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4299353623 00000100:00000001:0.0:1787829880.679663:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4299354880 00000100:00000001:0.0:1787829880.679664:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4299353623 00000100:00000001:0.0:1787829880.679666:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4299354880 00000100:00000001:0.0:1787829880.679668:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787829880.679669:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829880.679670:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787829880.679673:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829880/real 1787829880] req@ffff8d30c70143c0 x1874670646874240/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829905 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787829880.679679:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787829880.679680:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787829880.679681:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787829880.679683:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787829880.679684:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829880.679685:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787829880.679686:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787829880.679689:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c70143c0 x1874670646874240/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829905 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787829880.679693:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829880.679694:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787829880.679696:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829880.679697:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787829880.679697:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829880.679698:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787829880.679699:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787829880.679701:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829880.679703:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787829880.679704:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787829880.679707:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c70143c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646874240:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787829880.679709:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829880.679710:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787829880.679711:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787829880.679712:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787829880.679714:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787829880.679714:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787829880.679715:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787829880.679717:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787829880.679717:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787829880.679719:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787829880.679721:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829880.679723:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787829880.679724:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787829880.679725:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787829880.679727:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787829880.679729:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787829880.679730:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787829880.679732:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787829880.679733:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787829880.679735:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787829880.679736:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829880.679738:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829880.679742:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787829880.679743:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829880.679746:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787829880.679747:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787829880.679749:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1787829880.679750:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787829880.679752:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829880.679754:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787829880.679756:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787829880.679759:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787829880.679763:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c70152c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646874368:192.168.203.143@tcp:8:kworker.0 00000100:00000001:0.0:1787829880.679766:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787829880.679768:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787829880.679770:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829880.679771:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1787829880.679774:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829880.679781:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787829880.679783:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369540288 : -126234340011328 : ffff8d30c70152c0) 00000100:00000001:0.0:1787829880.679786:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1787829880.679806:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:0.0:1787829880.679809:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1787829880.679811:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787829880.679813:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829880.679814:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787829880.679816:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829880.679818:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4299353636 00000100:00000001:0.0:1787829880.679821:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4299354880 00000100:00000001:0.0:1787829880.679823:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4299353636 00000100:00000001:0.0:1787829880.679824:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4299354880 00000100:00000001:0.0:1787829880.679826:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787829880.679827:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829880.679829:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787829880.679831:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829880/real 1787829880] req@ffff8d30c70152c0 x1874670646874368/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829905 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787829880.679836:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787829880.679837:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787829880.679838:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787829880.679839:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787829880.679840:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829880.679842:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787829880.679843:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787829880.679845:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c70152c0 x1874670646874368/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829905 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787829880.679850:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829880.679852:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787829880.679853:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829880.679854:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787829880.679855:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829880.679856:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787829880.679857:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787829880.679859:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829880.679860:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787829880.679862:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787829880.679863:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c70152c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646874368:192.168.203.143@tcp:8:kworker.0 00000100:00000001:0.0:1787829880.679866:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829880.679867:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787829880.679868:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787829880.679869:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787829880.679871:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787829880.679872:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787829880.679873:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787829880.679874:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787829880.679875:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787829880.679877:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787829880.679878:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829880.679879:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829880.679881:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787829880.679882:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829880.679883:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787829880.679885:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829881.574737:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829881.574745:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829881.574838:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829881.574844:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829881.574848:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829882.598648:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829882.598653:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829882.598735:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829882.598740:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829882.598744:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829883.622734:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829883.622741:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829883.622825:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829883.622831:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829883.622834:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829884.646697:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829884.646702:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829884.646758:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829884.646763:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829884.646767:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829885.670701:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829885.670708:0:815:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1787829885.670710:0:815:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1787829885.670712:0:815:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829885.670717:0:815:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1787829885.670719:0:815:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829885.670721:0:815:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829885.670722:0:815:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829885.670754:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829885.670759:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829885.670763:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787829885.798696:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829885.798704:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1787829885.798706:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787829885.798709:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787829885.798713:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787829885.798719:0:40701:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 4686 00000100:00080000:3.0:1787829885.798728:0:40701:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787829885.798731:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829885.798733:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:3.0:1787829885.798738:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1787829885.798741:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1787829885.798744:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:3.0:1787829885.798747:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:3.0:1787829885.798749:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829885.798751:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:3.0:1787829885.798755:0:40701:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:3.0:1787829885.798758:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829885.798761:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787829885.798765:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829885.798769:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829885.798771:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:3.0:1787829885.798774:0:40701:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829885.798786:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787829885.798788:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829885.798791:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787829885.798794:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787829885.798796:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829885.798806:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829885.798813:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c70161c0 x1874670646874496/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787829885.798821:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787829885.798824:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829885.798838:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787829885.798840:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829885.798843:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1787829885.798844:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787829885.798845:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787829885.798847:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787829885.798849:0:40701:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4686 00000100:00080000:3.0:1787829885.798852:0:40701:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787829885.798854:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829885.798856:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:3.0:1787829885.798858:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1787829885.798859:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1787829885.798861:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000020:00000001:3.0:1787829885.798862:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:3.0:1787829885.798864:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829885.798865:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:0.0:1787829885.798868:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00080000:3.0:1787829885.798869:0:40701:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787829885.798873:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787829885.798873:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829885.798874:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:0.0:1787829885.798876:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1787829885.798879:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 02000000:00000001:3.0:1787829885.798880:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787829885.798881:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829885.798886:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829885.798888:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:3.0:1787829885.798889:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 02000000:00000001:0.0:1787829885.798893:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000008:00100000:3.0:1787829885.798895:0:40701:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 02000000:00000001:0.0:1787829885.798896:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:3.0:1787829885.798898:0:40701:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829885.798901:0:40701:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787829885.798904:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c70161c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646874496:192.168.203.143@tcp:250:kworker.0 02000000:00000001:3.0:1787829885.798905:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787829885.798908:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829885.798911:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:0.0:1787829885.798913:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1787829885.798914:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787829885.798917:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829885.798918:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:3.0:1787829885.798921:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829885.798922:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829885.798927:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00100000:3.0:1787829885.798931:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c7014b40 x1874670646874624/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:0.0:1787829885.798932:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829885.798954:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787829885.798956:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829885.798958:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829885.798961:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00080000:3.0:1787829885.798961:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1787829885.798964:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369544128 : -126234340007488 : ffff8d30c70161c0) 00000100:00080000:3.0:1787829885.798965:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1787829885.798969:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:3.0:1787829885.798970:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787829885.798973:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787829885.798975:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787829885.798977:0:40701:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4686 00000100:00080000:3.0:1787829885.798980:0:40701:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787829885.798982:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829885.798983:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:3.0:1787829885.798985:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1787829885.798986:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1787829885.798988:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000020:00000001:3.0:1787829885.798989:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:3.0:1787829885.798991:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829885.798992:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:3.0:1787829885.798994:0:40701:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:3.0:1787829885.798996:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829885.798997:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787829885.798999:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829885.799001:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829885.799002:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000800:00000100:0.0:1787829885.799006:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000008:00100000:3.0:1787829885.799006:0:40701:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1787829885.799011:0:40701:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829885.799013:0:40701:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829885.799017:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:0.0:1787829885.799018:0:2390:0:(events.c:34:request_out_callback()) Process entered 02000000:00000001:3.0:1787829885.799019:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829885.799022:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:0.0:1787829885.799024:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 02000000:00000001:3.0:1787829885.799024:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787829885.799028:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829885.799029:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829885.799032:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1787829885.799034:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829885.799037:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829885.799042:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c7014f00 x1874670646874752/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787829885.799044:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4299360000 00000100:00000001:3.0:1787829885.799052:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787829885.799053:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829885.799055:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 00000100:00000001:3.0:1787829885.799055:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787829885.799060:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829885.799063:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1787829885.799064:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4299360000 00000100:00000001:3.0:1787829885.799065:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787829885.799068:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787829885.799070:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 00000100:00000001:3.0:1787829885.799071:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787829885.799076:0:40701:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4686 00000100:00000001:0.0:1787829885.799077:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787829885.799081:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787829885.799082:0:40701:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787829885.799084:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00000001:3.0:1787829885.799085:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829885.799087:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:3.0:1787829885.799090:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000100:00100000:0.0:1787829885.799092:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829885/real 1787829885] req@ffff8d30c70161c0 x1874670646874496/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829910 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:3.0:1787829885.799093:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1787829885.799096:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000020:00000001:3.0:1787829885.799100:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:3.0:1787829885.799105:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829885.799107:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000400:00000001:0.0:1787829885.799112:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00080000:3.0:1787829885.799113:0:40701:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000400:00000001:0.0:1787829885.799115:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829885.799117:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829885.799120:0:2390:0:(events.c:78:reply_in_callback()) Process entered 02000000:00000001:3.0:1787829885.799120:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787829885.799125:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787829885.799126:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 02000000:00000001:3.0:1787829885.799127:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829885.799130:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1787829885.799131:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000008:00100000:3.0:1787829885.799132:0:40701:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000008:00000001:3.0:1787829885.799137:0:40701:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829885.799138:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829885.799138:0:40701:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829885.799142:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:3.0:1787829885.799143:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787829885.799145:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829885.799148:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787829885.799149:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787829885.799151:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787829885.799153:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c70161c0 x1874670646874496/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829910 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829885.799154:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829885.799161:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c7015680 x1874670646874880/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787829885.799178:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829885.799179:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787829885.799183:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829885.799183:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829885.799188:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829885.799188:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829885.799194:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787829885.799196:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829885.799198:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787829885.799200:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787829885.799202:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829885.799206:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787829885.799208:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787829885.799211:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c70161c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646874496:192.168.203.143@tcp:250:kworker.0 00000100:00000001:0.0:1787829885.799216:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829885.799218:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787829885.799220:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787829885.799222:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787829885.799225:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787829885.799226:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787829885.799228:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787829885.799231:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787829885.799232:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787829885.799234:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787829885.799235:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829885.799237:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829885.799244:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787829885.799245:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829885.799247:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787829885.799249:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787829885.799250:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1787829885.799252:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787829885.799253:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829885.799254:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787829885.799256:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787829885.799257:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787829885.799260:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c7014b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646874624:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787829885.799262:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787829885.799263:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787829885.799265:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829885.799266:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1787829885.799268:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829885.799273:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787829885.799274:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369538368 : -126234340013248 : ffff8d30c7014b40) 00000100:00000001:0.0:1787829885.799278:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1787829885.799287:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:0.0:1787829885.799290:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1787829885.799292:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787829885.799293:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829885.799294:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787829885.799296:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829885.799300:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4299358742 00000100:00000001:0.0:1787829885.799302:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4299360000 00000100:00000001:0.0:1787829885.799303:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4299358742 00000100:00000001:0.0:1787829885.799305:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4299360000 00000100:00000001:0.0:1787829885.799307:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787829885.799308:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829885.799309:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787829885.799312:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829885/real 1787829885] req@ffff8d30c7014b40 x1874670646874624/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829910 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787829885.799318:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787829885.799319:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787829885.799320:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787829885.799321:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787829885.799322:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829885.799323:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787829885.799324:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787829885.799327:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c7014b40 x1874670646874624/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829910 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787829885.799332:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829885.799333:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787829885.799334:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829885.799335:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787829885.799336:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829885.799337:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787829885.799338:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787829885.799340:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829885.799342:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787829885.799343:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787829885.799345:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c7014b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646874624:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787829885.799347:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1787829885.799348:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787829885.799349:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829885.799350:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787829885.799352:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787829885.799353:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787829885.799355:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c7014f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646874752:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787829885.799357:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787829885.799358:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787829885.799359:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829885.799360:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1787829885.799362:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829885.799364:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787829885.799365:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369539328 : -126234340012288 : ffff8d30c7014f00) 00000100:00000001:0.0:1787829885.799367:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1787829885.799373:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:0.0:1787829885.799376:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1787829885.799377:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787829885.799378:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829885.799379:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787829885.799380:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829885.799382:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4299358742 00000100:00000001:0.0:1787829885.799393:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4299360000 00000100:00000001:0.0:1787829885.799395:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4299358742 00000100:00000001:0.0:1787829885.799396:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4299360000 00000100:00000001:0.0:1787829885.799398:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787829885.799399:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829885.799400:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787829885.799403:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829885/real 1787829885] req@ffff8d30c7014f00 x1874670646874752/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829910 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787829885.799409:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787829885.799409:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787829885.799411:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787829885.799412:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787829885.799413:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829885.799414:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787829885.799415:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787829885.799418:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c7014f00 x1874670646874752/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829910 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787829885.799423:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829885.799424:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787829885.799425:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829885.799426:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787829885.799427:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829885.799428:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787829885.799429:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787829885.799430:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829885.799432:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787829885.799434:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787829885.799436:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c7014f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646874752:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787829885.799440:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1787829885.799441:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787829885.799442:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829885.799444:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787829885.799446:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787829885.799448:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787829885.799451:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c7015680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646874880:192.168.203.143@tcp:8:kworker.0 00000100:00000001:0.0:1787829885.799454:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787829885.799456:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787829885.799457:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829885.799459:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1787829885.799462:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829885.799468:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787829885.799470:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369541248 : -126234340010368 : ffff8d30c7015680) 00000100:00000001:0.0:1787829885.799474:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1787829885.799486:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:0.0:1787829885.799490:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1787829885.799493:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787829885.799495:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829885.799496:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787829885.799499:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829885.799501:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4299358755 00000100:00000001:0.0:1787829885.799503:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4299360000 00000100:00000001:0.0:1787829885.799504:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4299358755 00000100:00000001:0.0:1787829885.799506:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4299360000 00000100:00000001:0.0:1787829885.799507:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787829885.799509:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829885.799510:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787829885.799513:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829885/real 1787829885] req@ffff8d30c7015680 x1874670646874880/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829910 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787829885.799518:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787829885.799519:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787829885.799520:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787829885.799521:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787829885.799522:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829885.799523:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787829885.799524:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787829885.799527:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c7015680 x1874670646874880/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829910 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787829885.799531:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829885.799532:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787829885.799534:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829885.799535:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787829885.799536:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829885.799537:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787829885.799538:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787829885.799539:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829885.799541:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787829885.799542:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787829885.799545:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c7015680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646874880:192.168.203.143@tcp:8:kworker.0 00000100:00000001:0.0:1787829885.799547:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829885.799549:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787829885.799550:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787829885.799551:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787829885.799552:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787829885.799553:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787829885.799554:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787829885.799555:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787829885.799556:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787829885.799557:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787829885.799558:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829885.799559:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787829885.799560:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787829885.799561:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787829885.799562:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787829885.799563:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787829885.799564:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787829885.799565:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787829885.799565:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787829885.799567:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787829885.799567:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829885.799568:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787829885.799569:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787829885.799570:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787829885.799571:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787829885.799571:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787829885.799572:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787829885.799573:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787829885.799573:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787829885.799574:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787829885.799575:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829885.799576:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829885.799578:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787829885.799579:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829885.799580:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787829885.799581:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829886.694687:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829886.694692:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829886.694749:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829886.694752:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829886.694755:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829887.718737:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829887.718745:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829887.718813:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829887.718818:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829887.718821:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829888.742707:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829888.742715:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829888.742782:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829888.742786:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829888.742790:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829889.766711:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829889.766717:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829889.766766:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829889.766769:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829889.766772:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829890.790667:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829890.790672:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829890.790723:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829890.790726:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829890.790729:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787829890.918727:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829890.918738:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1787829890.918741:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787829890.918745:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787829890.918749:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787829890.918756:0:40701:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 4691 00000100:00080000:0.0:1787829890.918766:0:40701:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787829890.918769:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829890.918771:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787829890.918775:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787829890.918778:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787829890.918781:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:0.0:1787829890.918784:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:0.0:1787829890.918786:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829890.918788:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:0.0:1787829890.918793:0:40701:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787829890.918796:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829890.918798:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787829890.918802:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787829890.918806:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829890.918809:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1787829890.918811:0:40701:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829890.918822:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787829890.918824:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829890.918827:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787829890.918830:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787829890.918833:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829890.918843:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787829890.918852:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d252c0 x1874670646875008/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787829890.918863:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787829890.918867:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829890.918883:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787829890.918887:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829890.918892:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1787829890.918894:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787829890.918896:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787829890.918899:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787829890.918919:0:40701:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4691 00000100:00000001:2.0:1787829890.918922:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00080000:0.0:1787829890.918924:0:40701:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787829890.918926:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787829890.918928:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829890.918930:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1787829890.918932:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:2.0:1787829890.918933:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787829890.918937:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787829890.918938:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787829890.918941:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 02000000:00000001:2.0:1787829890.918943:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000020:00000001:0.0:1787829890.918944:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 02000000:00000001:2.0:1787829890.918946:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000020:00000001:0.0:1787829890.918949:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 02000000:00000001:2.0:1787829890.918950:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829890.918956:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00100000:2.0:1787829890.918958:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d252c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646875008:192.168.203.143@tcp:250:kworker.0 00000100:00000001:0.0:1787829890.918959:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:2.0:1787829890.918967:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00080000:0.0:1787829890.918969:0:40701:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 02000000:00000001:2.0:1787829890.918972:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:0.0:1787829890.918973:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829890.918974:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829890.918978:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:2.0:1787829890.918979:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1787829890.918985:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829890.918987:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787829890.918991:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829890.918993:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:2.0:1787829890.918997:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000008:00100000:0.0:1787829890.918999:0:40701:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00000001:2.0:1787829890.919000:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601341120 : -126234108210496 : ffff8d30d4d252c0) 00000008:00000001:0.0:1787829890.919006:0:40701:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829890.919006:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:0.0:1787829890.919010:0:40701:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829890.919015:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787829890.919017:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829890.919019:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787829890.919021:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787829890.919023:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829890.919028:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787829890.919035:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d25680 x1874670646875136/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787829890.919046:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829890.919052:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4299365120 00000100:00000001:0.0:1787829890.919053:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787829890.919057:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829890.919061:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1787829890.919063:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787829890.919069:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:2.0:1787829890.919070:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4299365120 00000100:00080000:0.0:1787829890.919077:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:2.0:1787829890.919078:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1787829890.919080:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:2.0:1787829890.919083:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00080000:0.0:1787829890.919084:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787829890.919086:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829890.919090:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:2.0:1787829890.919090:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829890.919094:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:0.0:1787829890.919097:0:40701:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4691 00000100:00080000:0.0:1787829890.919102:0:40701:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787829890.919103:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787829890.919107:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829890.919108:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:0.0:1787829890.919112:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:2.0:1787829890.919113:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787829890.919115:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1787829890.919118:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1787829890.919119:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787829890.919121:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829890.919123:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787829890.919124:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000020:00000001:0.0:1787829890.919126:0:40701:0:(genops.c:792:class_conn2export()) Process entered 02000000:00000001:2.0:1787829890.919127:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787829890.919131:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00100000:2.0:1787829890.919133:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d25680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646875136:192.168.203.143@tcp:38:kworker.0 00000020:00000001:0.0:1787829890.919135:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:2.0:1787829890.919138:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000020:00000001:0.0:1787829890.919141:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 02000000:00000001:2.0:1787829890.919141:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1787829890.919143:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829890.919147:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 02000000:00000001:2.0:1787829890.919147:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:0.0:1787829890.919152:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 02000000:00000001:2.0:1787829890.919152:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787829890.919161:0:40701:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787829890.919161:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787829890.919164:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601342080 : -126234108209536 : ffff8d30d4d25680) 00000100:00000001:0.0:1787829890.919167:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829890.919169:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 02000000:00000001:0.0:1787829890.919172:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787829890.919176:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787829890.919179:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829890.919181:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:2.0:1787829890.919181:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000008:00100000:0.0:1787829890.919186:0:40701:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00000001:2.0:1787829890.919187:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4299363862 00000100:00000001:2.0:1787829890.919192:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4299365120 00000008:00000001:0.0:1787829890.919193:0:40701:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829890.919196:0:40701:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829890.919202:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:2.0:1787829890.919203:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4299363862 02000000:00000001:0.0:1787829890.919206:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829890.919207:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4299365120 00000100:00000001:0.0:1787829890.919213:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:2.0:1787829890.919213:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787829890.919218:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:2.0:1787829890.919218:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829890.919224:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829890.919224:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829890.919230:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829890.919232:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787829890.919234:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829890.919234:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:2.0:1787829890.919240:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787829890.919242:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1787829890.919244:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787829890.919246:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d27c00 x1874670646875264/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787829890.919246:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829890.919267:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787829890.919269:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829890.919278:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787829890.919282:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829890.919285:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1787829890.919286:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787829890.919288:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787829890.919290:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787829890.919294:0:40701:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4691 00000100:00080000:0.0:1787829890.919298:0:40701:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787829890.919301:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829890.919303:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787829890.919306:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787829890.919307:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:2.0:1787829890.919309:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:0.0:1787829890.919310:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:2.0:1787829890.919313:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000020:00000001:0.0:1787829890.919315:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:2.0:1787829890.919317:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1787829890.919319:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829890.919321:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787829890.919323:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:2.0:1787829890.919324:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829890.919329:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000100:00080000:0.0:1787829890.919330:0:40701:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 02000000:00000001:2.0:1787829890.919334:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787829890.919336:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829890.919339:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829890.919345:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00100000:2.0:1787829890.919346:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d27c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646875264:192.168.203.143@tcp:38:kworker.0 02000000:00000001:0.0:1787829890.919351:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787829890.919353:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787829890.919356:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829890.919357:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:0.0:1787829890.919360:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 02000000:00000001:2.0:1787829890.919361:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000008:00100000:0.0:1787829890.919364:0:40701:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 02000000:00000001:2.0:1787829890.919365:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000008:00000001:0.0:1787829890.919370:0:40701:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829890.919371:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829890.919373:0:40701:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829890.919380:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787829890.919382:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829890.919382:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787829890.919386:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601351680 : -126234108199936 : ffff8d30d4d27c00) 00000100:00000001:0.0:1787829890.919387:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787829890.919390:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787829890.919393:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829890.919394:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:0.0:1787829890.919402:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787829890.919409:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d26940 x1874670646875392/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787829890.919411:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829890.919416:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4299363862 00000100:00000001:2.0:1787829890.919421:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4299365120 00000100:00000001:0.0:1787829890.919424:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787829890.919425:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4299363862 00000100:00000001:2.0:1787829890.919429:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4299365120 00000100:00000001:0.0:1787829890.919430:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829890.919431:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787829890.919436:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829890.919437:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829890.919440:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829890.919444:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829890.919448:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1787829890.919449:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:2.0:1787829890.919452:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787829890.919453:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1787829890.919455:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1787829890.919457:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787829890.919459:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829890.919462:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787829890.919464:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787829890.919466:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787829890.919472:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d26940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646875392:192.168.203.143@tcp:8:kworker.0 00000100:00000001:2.0:1787829890.919476:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787829890.919478:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1787829890.919480:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829890.919481:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1787829890.919485:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829890.919491:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787829890.919492:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601346880 : -126234108204736 : ffff8d30d4d26940) 00000100:00000001:2.0:1787829890.919496:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:2.0:1787829890.919507:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829890.919511:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4299363875 00000100:00000001:2.0:1787829890.919514:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4299365120 00000100:00000001:2.0:1787829890.919516:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4299363875 00000100:00000001:2.0:1787829890.919518:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4299365120 00000100:00000001:2.0:1787829890.919520:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787829890.919522:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829890.919525:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829890.919527:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829890.919530:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1787829890.919531:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=5 : 5 : 5) 00000100:00000001:2.0:1787829890.919534:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787829890.919535:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1787829890.919536:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829890.919538:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829891.814705:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829891.814710:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829891.814773:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829891.814777:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829891.814780:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829892.838676:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829892.838682:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829892.838718:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829892.838721:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829892.838724:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829893.862752:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829893.862761:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829893.862839:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829893.862843:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829893.862845:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:2.0:1787829893.990934:0:2379:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.143:988 00000400:00000100:2.0:1787829893.990980:0:2379:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.143@tcp at host 192.168.203.143 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1787829893.991000:0:2379:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.143@tcp 00000100:00000001:2.0:1787829893.991019:0:2379:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1787829893.991041:0:2379:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787829893.991045:0:2379:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829893.991049:0:2379:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:2.0:1787829893.991060:0:2379:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.143@tcp 00000100:00000001:2.0:1787829893.991065:0:2379:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1787829893.991068:0:2379:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787829893.991069:0:2379:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829893.991071:0:2379:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:2.0:1787829893.991076:0:2379:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.143@tcp 00000100:00000001:2.0:1787829893.991082:0:2379:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1787829893.991084:0:2379:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787829893.991086:0:2379:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829893.991089:0:2379:0:(events.c:68:request_out_callback()) Process leaving 00000800:00000100:2.0:1787829893.991093:0:2379:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.143@tcp 00000100:00000001:2.0:1787829893.991097:0:2379:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1787829893.991100:0:2379:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787829893.991101:0:2379:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829893.991103:0:2379:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787829893.991362:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787829893.991368:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787829893.991371:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787829893.991386:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829890/real 1787829893] req@ffff8d30d4d26940 x1874670646875392/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829915 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787829893.991403:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787829893.991406:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787829893.991416:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787829893.991419:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787829893.991421:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829893.991424:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787829893.991426:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787829893.991431:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d26940 x1874670646875392/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829915 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787829893.991439:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829893.991443:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787829893.991447:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829893.991448:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787829893.991450:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829893.991454:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787829893.991457:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787829893.991464:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829893.991470:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787829893.991474:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787829893.991479:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d26940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646875392:192.168.203.143@tcp:8:kworker.0 00000100:00000001:0.0:1787829893.991485:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787829893.991502:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829890/real 1787829893] req@ffff8d30d4d27c00 x1874670646875264/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829915 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787829893.991513:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787829893.991515:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787829893.991521:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787829893.991523:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787829893.991525:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829893.991526:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787829893.991528:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787829893.991531:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d27c00 x1874670646875264/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829915 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787829893.991538:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829893.991540:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787829893.991543:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829893.991544:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787829893.991546:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829893.991547:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787829893.991549:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787829893.991552:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829893.991555:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787829893.991557:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787829893.991561:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d27c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646875264:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787829893.991568:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787829893.991572:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829890/real 1787829893] req@ffff8d30d4d25680 x1874670646875136/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829915 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787829893.991580:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787829893.991582:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787829893.991586:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787829893.991588:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787829893.991590:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829893.991592:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787829893.991593:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787829893.991597:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d25680 x1874670646875136/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829915 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787829893.991654:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829893.991657:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787829893.991659:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829893.991661:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787829893.991662:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829893.991664:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787829893.991666:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787829893.991669:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829893.991675:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787829893.991678:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787829893.991681:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d25680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646875136:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787829893.991689:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787829893.991696:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829890/real 1787829893] req@ffff8d30d4d252c0 x1874670646875008/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829915 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787829893.991719:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787829893.991723:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787829893.991731:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787829893.991736:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787829893.991740:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829893.991742:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787829893.991744:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787829893.991750:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d252c0 x1874670646875008/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829915 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787829893.991757:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829893.991759:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787829893.991763:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829893.991765:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787829893.991767:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829893.991769:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787829893.991772:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787829893.991776:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829893.991780:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787829893.991783:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787829893.991787:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d252c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646875008:192.168.203.143@tcp:250:kworker.0 00000100:00000001:0.0:1787829893.991794:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829893.991797:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787829893.991799:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787829893.991813:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787829893.991819:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787829893.991823:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787829893.991825:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787829893.991829:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787829893.991831:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787829893.991835:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787829893.991836:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829893.991838:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787829893.991840:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787829893.991841:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787829893.991855:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787829893.991856:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787829893.991858:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787829893.991861:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787829893.991862:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787829893.991865:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787829893.991866:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829893.991868:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787829893.991870:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787829893.991871:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787829893.991873:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787829893.991874:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787829893.991875:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787829893.991877:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787829893.991878:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787829893.992975:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787829893.992979:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829893.992982:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787829893.992984:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787829893.992987:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787829893.992991:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787829893.992992:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787829893.992994:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787829893.992998:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787829893.993000:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787829893.993002:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787829893.993004:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829893.993006:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829893.993014:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787829893.993016:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829893.993018:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787829893.993020:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829894.886659:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829894.886664:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829894.886692:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829894.886696:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829894.886698:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829895.910710:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829895.910716:0:815:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1787829895.910718:0:815:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1787829895.910721:0:815:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829895.910724:0:815:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1787829895.910726:0:815:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829895.910728:0:815:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829895.910730:0:815:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829895.910790:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829895.910796:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829895.910800:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787829896.038777:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829896.038789:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1787829896.038793:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787829896.038797:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787829896.038803:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787829896.038813:0:40701:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 4696 00000100:00080000:0.0:1787829896.038827:0:40701:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787829896.038831:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829896.038833:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787829896.038838:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787829896.038842:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787829896.038855:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:0.0:1787829896.038859:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:0.0:1787829896.038864:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829896.038881:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:0.0:1787829896.038888:0:40701:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787829896.038891:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829896.038894:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787829896.038899:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787829896.038903:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829896.038906:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1787829896.038909:0:40701:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829896.038921:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787829896.038923:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829896.038927:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787829896.038931:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787829896.038936:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829896.038951:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787829896.038964:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d252c0 x1874670646875520/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787829896.038974:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787829896.038979:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829896.038999:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787829896.039002:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829896.039006:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1787829896.039008:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787829896.039010:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787829896.039013:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787829896.039018:0:40701:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4696 00000100:00080000:0.0:1787829896.039023:0:40701:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787829896.039026:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829896.039028:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787829896.039031:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787829896.039033:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787829896.039036:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000020:00000001:0.0:1787829896.039039:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:0.0:1787829896.039041:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829896.039044:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787829896.039045:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:0.0:1787829896.039052:0:40701:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787829896.039055:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829896.039057:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:2.0:1787829896.039066:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 02000000:00000001:0.0:1787829896.039067:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787829896.039072:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 02000000:00000001:0.0:1787829896.039073:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829896.039079:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:2.0:1787829896.039081:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000008:00100000:0.0:1787829896.039086:0:40701:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00000001:2.0:1787829896.039090:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:0.0:1787829896.039102:0:40701:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829896.039106:0:40701:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829896.039106:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787829896.039117:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787829896.039118:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787829896.039123:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829896.039127:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829896.039129:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787829896.039133:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787829896.039138:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787829896.039145:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d252c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646875520:192.168.203.143@tcp:250:kworker.0 00000100:00000001:0.0:1787829896.039148:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829896.039160:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00100000:0.0:1787829896.039164:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d25680 x1874670646875648/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:2.0:1787829896.039168:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1787829896.039179:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829896.039188:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 02000000:00000001:2.0:1787829896.039190:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:0.0:1787829896.039196:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829896.039201:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829896.039205:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787829896.039206:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829896.039217:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1787829896.039218:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787829896.039220:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787829896.039224:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787829896.039228:0:40701:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4696 00000100:00000001:2.0:1787829896.039233:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00080000:0.0:1787829896.039235:0:40701:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787829896.039238:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601341120 : -126234108210496 : ffff8d30d4d252c0) 00000100:00000001:0.0:1787829896.039241:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829896.039245:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:2.0:1787829896.039246:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000020:00000001:0.0:1787829896.039253:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787829896.039255:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787829896.039258:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000020:00000001:0.0:1787829896.039261:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:0.0:1787829896.039263:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829896.039266:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:0.0:1787829896.039271:0:40701:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787829896.039273:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829896.039275:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787829896.039280:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787829896.039283:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829896.039285:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1787829896.039287:0:40701:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1787829896.039290:0:40701:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829896.039292:0:40701:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829896.039298:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000800:00000100:2.0:1787829896.039300:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 02000000:00000001:0.0:1787829896.039302:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829896.039308:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787829896.039309:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787829896.039313:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829896.039320:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1787829896.039321:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829896.039333:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00100000:0.0:1787829896.039334:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d27c00 x1874670646875776/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787829896.039341:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829896.039352:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787829896.039361:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787829896.039361:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829896.039369:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829896.039373:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4299370240 00000100:00000001:0.0:1787829896.039375:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787829896.039385:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829896.039390:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1787829896.039392:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:2.0:1787829896.039394:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 00000100:00080000:0.0:1787829896.039396:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787829896.039402:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787829896.039409:0:40701:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4696 00000100:00000001:2.0:1787829896.039410:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4299370240 00000100:00080000:0.0:1787829896.039419:0:40701:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787829896.039421:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1787829896.039424:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829896.039430:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:2.0:1787829896.039430:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000020:00000001:0.0:1787829896.039436:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:2.0:1787829896.039439:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787829896.039441:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787829896.039444:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:2.0:1787829896.039447:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000020:00000001:0.0:1787829896.039451:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:0.0:1787829896.039456:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829896.039458:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00100000:2.0:1787829896.039463:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829896/real 1787829896] req@ffff8d30d4d252c0 x1874670646875520/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829921 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829896.039466:0:40701:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787829896.039477:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829896.039483:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787829896.039491:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000400:00000001:2.0:1787829896.039496:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 02000000:00000001:0.0:1787829896.039497:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:2.0:1787829896.039502:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787829896.039503:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1787829896.039511:0:40701:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00000001:2.0:1787829896.039513:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000008:00000001:0.0:1787829896.039515:0:40701:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829896.039520:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787829896.039521:0:40701:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829896.039525:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:0.0:1787829896.039530:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:2.0:1787829896.039532:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 02000000:00000001:0.0:1787829896.039535:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829896.039538:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829896.039540:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787829896.039545:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787829896.039548:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787829896.039551:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d252c0 x1874670646875520/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829921 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787829896.039556:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787829896.039567:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d26940 x1874670646875904/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787829896.039571:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829896.039587:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787829896.039592:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829896.039598:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787829896.039637:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787829896.039641:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829896.039645:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829896.039673:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829896.039679:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787829896.039681:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787829896.039686:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829896.039693:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:2.0:1787829896.039696:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1787829896.039700:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d252c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646875520:192.168.203.143@tcp:250:kworker.0 00000100:00000001:2.0:1787829896.039706:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829896.039709:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787829896.039710:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787829896.039713:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1787829896.039717:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787829896.039720:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:2.0:1787829896.039722:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:2.0:1787829896.039726:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787829896.039728:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1787829896.039733:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787829896.039734:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829896.039737:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829896.039744:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1787829896.039747:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829896.039749:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787829896.039751:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1787829896.039753:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1787829896.039755:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787829896.039756:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829896.039759:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787829896.039761:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787829896.039763:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787829896.039767:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d25680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646875648:192.168.203.143@tcp:38:kworker.0 00000100:00000001:2.0:1787829896.039772:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787829896.039775:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1787829896.039777:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829896.039778:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1787829896.039782:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829896.039790:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787829896.039791:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601342080 : -126234108209536 : ffff8d30d4d25680) 00000100:00000001:2.0:1787829896.039796:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1787829896.039813:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:2.0:1787829896.039818:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1787829896.039820:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787829896.039822:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829896.039824:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1787829896.039827:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829896.039832:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4299368983 00000100:00000001:2.0:1787829896.039835:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4299370240 00000100:00000001:2.0:1787829896.039838:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4299368983 00000100:00000001:2.0:1787829896.039840:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4299370240 00000100:00000001:2.0:1787829896.039842:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787829896.039844:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829896.039846:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1787829896.039863:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829896/real 1787829896] req@ffff8d30d4d25680 x1874670646875648/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829921 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1787829896.039888:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787829896.039890:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1787829896.039893:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1787829896.039895:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787829896.039897:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829896.039899:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787829896.039900:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787829896.039905:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d25680 x1874670646875648/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829921 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787829896.039912:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829896.039914:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787829896.039917:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829896.039919:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787829896.039921:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829896.039925:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787829896.039927:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787829896.039930:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829896.039934:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:2.0:1787829896.039936:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1787829896.039939:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d25680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646875648:192.168.203.143@tcp:38:kworker.0 00000100:00000001:2.0:1787829896.039952:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1787829896.039954:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787829896.039955:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829896.039958:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787829896.039960:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787829896.039963:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787829896.039967:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d27c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646875776:192.168.203.143@tcp:38:kworker.0 00000100:00000001:2.0:1787829896.039971:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787829896.039973:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1787829896.039976:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829896.039978:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1787829896.039981:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829896.039987:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787829896.039989:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601351680 : -126234108199936 : ffff8d30d4d27c00) 00000100:00000001:2.0:1787829896.039994:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1787829896.040013:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:2.0:1787829896.040019:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1787829896.040022:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787829896.040024:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829896.040025:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1787829896.040028:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829896.040032:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4299368983 00000100:00000001:2.0:1787829896.040035:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4299370240 00000100:00000001:2.0:1787829896.040037:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4299368983 00000100:00000001:2.0:1787829896.040040:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4299370240 00000100:00000001:2.0:1787829896.040041:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787829896.040044:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829896.040045:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1787829896.040050:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829896/real 1787829896] req@ffff8d30d4d27c00 x1874670646875776/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829921 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1787829896.040058:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787829896.040060:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1787829896.040063:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1787829896.040065:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787829896.040066:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829896.040068:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787829896.040069:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787829896.040073:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d27c00 x1874670646875776/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829921 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787829896.040080:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829896.040083:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787829896.040085:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829896.040087:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787829896.040088:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829896.040090:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787829896.040091:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787829896.040094:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829896.040098:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:2.0:1787829896.040100:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1787829896.040104:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d27c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646875776:192.168.203.143@tcp:38:kworker.0 00000100:00000001:2.0:1787829896.040108:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1787829896.040109:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787829896.040111:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829896.040114:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787829896.040116:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787829896.040118:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787829896.040121:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d26940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646875904:192.168.203.143@tcp:8:kworker.0 00000100:00000001:2.0:1787829896.040125:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787829896.040127:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1787829896.040129:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829896.040131:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1787829896.040134:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829896.040140:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787829896.040142:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601346880 : -126234108204736 : ffff8d30d4d26940) 00000100:00000001:2.0:1787829896.040146:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1787829896.040159:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:2.0:1787829896.040164:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1787829896.040166:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787829896.040167:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829896.040169:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1787829896.040172:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829896.040176:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4299368996 00000100:00000001:2.0:1787829896.040178:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4299370240 00000100:00000001:2.0:1787829896.040181:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4299368996 00000100:00000001:2.0:1787829896.040183:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4299370240 00000100:00000001:2.0:1787829896.040185:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787829896.040188:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829896.040190:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1787829896.040194:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829896/real 1787829896] req@ffff8d30d4d26940 x1874670646875904/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829921 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1787829896.040202:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787829896.040204:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1787829896.040207:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1787829896.040208:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787829896.040209:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829896.040210:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787829896.040211:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787829896.040214:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d26940 x1874670646875904/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829921 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787829896.040219:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829896.040221:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787829896.040222:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829896.040223:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787829896.040224:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829896.040225:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787829896.040227:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787829896.040229:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829896.040231:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:2.0:1787829896.040233:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1787829896.040235:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d26940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646875904:192.168.203.143@tcp:8:kworker.0 00000100:00000001:2.0:1787829896.040237:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829896.040238:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787829896.040239:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787829896.040240:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1787829896.040242:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787829896.040243:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:2.0:1787829896.040244:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:2.0:1787829896.040246:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787829896.040247:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1787829896.040249:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787829896.040249:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829896.040251:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787829896.040252:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787829896.040252:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1787829896.040253:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787829896.040254:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:2.0:1787829896.040255:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:2.0:1787829896.040256:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787829896.040257:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1787829896.040259:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787829896.040260:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829896.040261:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787829896.040262:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787829896.040262:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1787829896.040263:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787829896.040264:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:2.0:1787829896.040265:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:2.0:1787829896.040265:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787829896.040266:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1787829896.040268:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787829896.040269:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829896.040270:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829896.040273:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1787829896.040275:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829896.040277:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787829896.040279:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829896.934727:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829896.934734:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829896.934788:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829896.934791:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829896.934794:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829897.958681:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829897.958686:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829897.958739:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829897.958742:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829897.958744:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829898.982713:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829898.982720:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829898.982778:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829898.982782:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829898.982784:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829900.006668:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829900.006672:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829900.006725:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829900.006728:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829900.006730:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829901.030702:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829901.030708:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829901.030763:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829901.030766:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829901.030769:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787829901.158735:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829901.158743:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1787829901.158746:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787829901.158750:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787829901.158755:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787829901.158762:0:40701:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 4701 00000100:00080000:0.0:1787829901.158772:0:40701:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787829901.158775:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829901.158777:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787829901.158781:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787829901.158784:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787829901.158788:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:0.0:1787829901.158791:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:0.0:1787829901.158793:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829901.158794:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:0.0:1787829901.158799:0:40701:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787829901.158801:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829901.158803:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787829901.158806:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787829901.158809:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829901.158812:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1787829901.158815:0:40701:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829901.158823:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787829901.158825:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829901.158828:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787829901.158845:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787829901.158849:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829901.158860:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787829901.158869:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d26940 x1874670646876032/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787829901.158879:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787829901.158885:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829901.158900:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787829901.158903:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829901.158907:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1787829901.158909:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787829901.158911:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787829901.158914:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787829901.158917:0:40701:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4701 00000100:00080000:0.0:1787829901.158921:0:40701:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787829901.158924:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829901.158925:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787829901.158928:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787829901.158930:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:3.0:1787829901.158931:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:0.0:1787829901.158932:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000020:00000001:0.0:1787829901.158936:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:3.0:1787829901.158936:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787829901.158938:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1787829901.158942:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829901.158942:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787829901.158946:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:3.0:1787829901.158946:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787829901.158951:0:40701:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787829901.158956:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829901.158956:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787829901.158959:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787829901.158960:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787829901.158966:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787829901.158969:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829901.158971:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1787829901.158975:0:40701:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1787829901.158979:0:40701:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829901.158981:0:40701:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829901.158993:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787829901.158995:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829901.158997:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787829901.158998:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787829901.159000:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829901.159004:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787829901.159010:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d27c00 x1874670646876160/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787829901.159018:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787829901.159020:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829901.159023:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787829901.159025:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829901.159029:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1787829901.159030:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787829901.159031:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787829901.159034:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787829901.159036:0:40701:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4701 00000100:00080000:0.0:1787829901.159040:0:40701:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787829901.159042:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829901.159044:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787829901.159046:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787829901.159048:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787829901.159050:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000020:00000001:0.0:1787829901.159052:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:0.0:1787829901.159055:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829901.159056:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:0.0:1787829901.159060:0:40701:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787829901.159062:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829901.159064:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787829901.159067:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787829901.159069:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829901.159070:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1787829901.159072:0:40701:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1787829901.159075:0:40701:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829901.159076:0:40701:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829901.159080:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787829901.159081:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829901.159083:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787829901.159084:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787829901.159086:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829901.159090:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787829901.159094:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d25680 x1874670646876288/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787829901.159108:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787829901.159110:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829901.159112:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787829901.159115:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829901.159118:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1787829901.159119:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787829901.159120:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787829901.159122:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787829901.159125:0:40701:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4701 00000100:00080000:0.0:1787829901.159128:0:40701:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787829901.159130:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829901.159132:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787829901.159134:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787829901.159135:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787829901.159137:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000020:00000001:0.0:1787829901.159140:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:0.0:1787829901.159142:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829901.159143:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:0.0:1787829901.159146:0:40701:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787829901.159148:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829901.159150:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787829901.159152:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787829901.159153:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829901.159155:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1787829901.159157:0:40701:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1787829901.159160:0:40701:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829901.159161:0:40701:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829901.159165:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787829901.159166:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829901.159168:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787829901.159169:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787829901.159170:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829901.159175:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787829901.159179:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d252c0 x1874670646876416/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787829901.159185:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 02000000:00000001:3.0:1787829901.159189:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829901.159190:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829901.159194:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829901.159205:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d26940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646876032:192.168.203.143@tcp:250:kworker.0 00000100:00000001:3.0:1787829901.159210:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1787829901.159262:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1787829901.159265:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829901.159267:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1787829901.159271:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829901.159285:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787829901.159288:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601346880 : -126234108204736 : ffff8d30d4d26940) 00000100:00000001:3.0:1787829901.159291:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1787829901.159332:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:3.0:1787829901.159342:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787829901.159346:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829901.159353:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829901.159355:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1787829901.159360:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829901.159363:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4299375360 00000100:00000001:3.0:1787829901.159371:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 00000100:00000001:3.0:1787829901.159381:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4299375360 00000100:00000001:3.0:1787829901.159386:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 00000100:00000001:3.0:1787829901.159388:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829901.159391:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829901.159393:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1787829901.159398:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829901/real 1787829901] req@ffff8d30d4d26940 x1874670646876032/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829926 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1787829901.159408:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787829901.159410:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829901.159413:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829901.159416:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829901.159417:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829901.159419:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829901.159422:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829901.159425:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d26940 x1874670646876032/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829926 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829901.159432:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829901.159435:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829901.159438:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829901.159439:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829901.159441:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829901.159443:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829901.159445:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829901.159448:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829901.159451:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829901.159454:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829901.159457:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d26940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646876032:192.168.203.143@tcp:250:kworker.0 00000100:00000001:3.0:1787829901.159462:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829901.159464:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829901.159466:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829901.159468:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829901.159470:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829901.159472:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829901.159474:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829901.159478:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829901.159479:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829901.159482:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829901.159483:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829901.159485:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829901.159491:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787829901.159494:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829901.159496:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1787829901.159497:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:3.0:1787829901.159498:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787829901.159500:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787829901.159501:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829901.159502:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787829901.159505:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829901.159507:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829901.159509:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d27c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646876160:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829901.159512:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1787829901.159514:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1787829901.159516:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829901.159517:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1787829901.159520:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829901.159525:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787829901.159527:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601351680 : -126234108199936 : ffff8d30d4d27c00) 00000100:00000001:3.0:1787829901.159531:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1787829901.159542:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:3.0:1787829901.159546:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787829901.159548:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829901.159549:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829901.159551:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1787829901.159553:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829901.159557:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4299374102 00000100:00000001:3.0:1787829901.159560:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4299375360 00000100:00000001:3.0:1787829901.159561:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4299374102 00000100:00000001:3.0:1787829901.159564:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4299375360 00000100:00000001:3.0:1787829901.159566:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829901.159568:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829901.159569:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1787829901.159573:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829901/real 1787829901] req@ffff8d30d4d27c00 x1874670646876160/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829926 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1787829901.159580:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787829901.159581:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829901.159583:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829901.159584:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829901.159585:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829901.159587:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829901.159588:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829901.159592:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d27c00 x1874670646876160/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829926 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829901.159597:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829901.159621:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829901.159624:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829901.159625:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829901.159626:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829901.159628:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829901.159631:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829901.159633:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829901.159637:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829901.159639:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829901.159642:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d27c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646876160:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829901.159646:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787829901.159647:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787829901.159649:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829901.159651:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787829901.159654:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829901.159655:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829901.159658:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d25680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646876288:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829901.159661:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1787829901.159662:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1787829901.159664:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829901.159666:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1787829901.159668:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829901.159672:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787829901.159674:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601342080 : -126234108209536 : ffff8d30d4d25680) 00000100:00000001:3.0:1787829901.159677:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1787829901.159687:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:3.0:1787829901.159692:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787829901.159693:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829901.159694:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829901.159696:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1787829901.159698:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829901.159701:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4299374103 00000100:00000001:3.0:1787829901.159702:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4299375360 00000100:00000001:3.0:1787829901.159704:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4299374103 00000100:00000001:3.0:1787829901.159706:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4299375360 00000100:00000001:3.0:1787829901.159707:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829901.159709:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829901.159710:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1787829901.159714:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829901/real 1787829901] req@ffff8d30d4d25680 x1874670646876288/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829926 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1787829901.159721:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787829901.159722:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829901.159724:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829901.159726:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829901.159727:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829901.159729:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829901.159730:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829901.159734:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d25680 x1874670646876288/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829926 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829901.159740:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829901.159742:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829901.159744:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829901.159745:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829901.159746:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829901.159748:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829901.159749:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829901.159751:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829901.159754:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829901.159765:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829901.159769:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d25680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646876288:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829901.159775:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:3.0:1787829901.159777:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787829901.159778:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829901.159780:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787829901.159781:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829901.159783:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829901.159786:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d252c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646876416:192.168.203.143@tcp:8:kworker.0 00000100:00000001:3.0:1787829901.159789:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1787829901.159791:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1787829901.159793:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829901.159794:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1787829901.159797:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829901.159802:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787829901.159803:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601341120 : -126234108210496 : ffff8d30d4d252c0) 00000100:00000001:3.0:1787829901.159807:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:3.0:1787829901.159820:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:3.0:1787829901.159824:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:3.0:1787829901.159826:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829901.159828:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829901.159846:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:3.0:1787829901.159849:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829901.159853:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4299374116 00000100:00000001:3.0:1787829901.159855:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4299375360 00000100:00000001:3.0:1787829901.159858:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4299374116 00000100:00000001:3.0:1787829901.159860:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4299375360 00000100:00000001:3.0:1787829901.159862:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829901.159864:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829901.159866:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:3.0:1787829901.159871:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829901/real 1787829901] req@ffff8d30d4d252c0 x1874670646876416/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829926 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:3.0:1787829901.159878:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:3.0:1787829901.159879:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829901.159882:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829901.159883:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:3.0:1787829901.159885:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829901.159886:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829901.159887:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829901.159891:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d252c0 x1874670646876416/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829926 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829901.159897:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829901.159899:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829901.159901:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829901.159902:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829901.159903:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829901.159904:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:3.0:1787829901.159906:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:3.0:1787829901.159908:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1787829901.159911:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829901.159913:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:3.0:1787829901.159916:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d252c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646876416:192.168.203.143@tcp:8:kworker.0 00000100:00000001:3.0:1787829901.159919:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829901.159921:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829901.159922:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829901.159924:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829901.159926:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829901.159927:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829901.159929:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829901.159930:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829901.159931:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829901.159934:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829901.159935:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829901.159937:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829901.159938:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829901.159939:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829901.159940:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829901.159941:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829901.159942:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829901.159943:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829901.159944:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829901.159946:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829901.159947:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829901.159949:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829901.159950:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:3.0:1787829901.159951:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:3.0:1787829901.159952:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:3.0:1787829901.159953:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:3.0:1787829901.159954:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829901.159956:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:3.0:1787829901.159957:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829901.159958:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:3.0:1787829901.159959:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829901.159961:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829901.159964:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787829901.159965:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829901.159967:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1787829901.159968:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829902.054710:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829902.054715:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829902.054763:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829902.054767:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829902.054770:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829903.078707:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829903.078711:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829903.078763:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829903.078766:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829903.078769:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829904.102804:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829904.102824:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829904.102882:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829904.102887:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829904.102889:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829905.126724:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829905.126731:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829905.126784:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829905.126788:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829905.126791:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829906.150703:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829906.150712:0:815:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1787829906.150714:0:815:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1787829906.150718:0:815:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829906.150722:0:815:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1787829906.150725:0:815:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829906.150727:0:815:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829906.150729:0:815:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829906.150790:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829906.150811:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829906.150816:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787829906.278707:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829906.278715:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1787829906.278716:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787829906.278719:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787829906.278722:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787829906.278728:0:40701:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 4706 00000100:00080000:3.0:1787829906.278736:0:40701:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787829906.278738:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829906.278739:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:3.0:1787829906.278742:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1787829906.278744:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1787829906.278746:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:3.0:1787829906.278748:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:3.0:1787829906.278749:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829906.278750:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:3.0:1787829906.278753:0:40701:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:3.0:1787829906.278754:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829906.278756:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787829906.278758:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829906.278760:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829906.278761:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:3.0:1787829906.278762:0:40701:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829906.278771:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787829906.278772:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829906.278774:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787829906.278776:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787829906.278779:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829906.278787:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829906.278793:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c7016580 x1874670646876544/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787829906.278813:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787829906.278816:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829906.278827:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787829906.278829:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829906.278831:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1787829906.278832:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787829906.278833:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787829906.278835:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787829906.278837:0:40701:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4706 00000100:00080000:3.0:1787829906.278839:0:40701:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787829906.278840:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829906.278841:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:3.0:1787829906.278843:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1787829906.278843:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1787829906.278845:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000020:00000001:3.0:1787829906.278846:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:3.0:1787829906.278847:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829906.278848:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:3.0:1787829906.278851:0:40701:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:3.0:1787829906.278853:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829906.278854:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:2.0:1787829906.278859:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:3.0:1787829906.278859:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829906.278860:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829906.278863:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:2.0:1787829906.278864:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1787829906.278867:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000008:00100000:3.0:1787829906.278867:0:40701:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00000001:2.0:1787829906.278869:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000008:00000001:3.0:1787829906.278871:0:40701:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829906.278872:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829906.278874:0:40701:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829906.278876:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787829906.278878:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787829906.278879:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829906.278880:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829906.278883:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829906.278884:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787829906.278887:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00100000:2.0:1787829906.278890:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c7016580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646876544:192.168.203.143@tcp:250:kworker.0 02000000:00000001:3.0:1787829906.278891:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829906.278896:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829906.278898:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00100000:3.0:1787829906.278902:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c7015a40 x1874670646876672/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:2.0:1787829906.278903:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1787829906.278906:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829906.278910:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:3.0:1787829906.278913:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 02000000:00000001:2.0:1787829906.278915:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829906.278915:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829906.278920:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787829906.278923:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829906.278925:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1787829906.278927:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787829906.278928:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787829906.278930:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787829906.278932:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369545088 : -126234340006528 : ffff8d30c7016580) 00000100:00000001:3.0:1787829906.278932:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:2.0:1787829906.278937:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00080000:3.0:1787829906.278938:0:40701:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4706 00000100:00080000:3.0:1787829906.278943:0:40701:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787829906.278945:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829906.278947:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:3.0:1787829906.278949:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1787829906.278950:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1787829906.278952:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000020:00000001:3.0:1787829906.278955:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:3.0:1787829906.278956:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829906.278958:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:3.0:1787829906.278962:0:40701:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:3.0:1787829906.278964:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829906.278966:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787829906.278969:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000800:00000100:2.0:1787829906.278973:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 02000000:00000001:3.0:1787829906.278974:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829906.278977:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:3.0:1787829906.278978:0:40701:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1787829906.278981:0:40701:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829906.278982:0:40701:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829906.278984:0:2390:0:(events.c:34:request_out_callback()) Process entered 02000000:00000001:3.0:1787829906.278987:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:2.0:1787829906.278988:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 02000000:00000001:3.0:1787829906.278989:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829906.278991:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829906.278992:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787829906.278994:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:2.0:1787829906.278995:0:2390:0:(events.c:68:request_out_callback()) Process leaving 02000000:00000001:3.0:1787829906.278996:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829906.279000:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829906.279002:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829906.279004:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4299380480 00000100:00100000:3.0:1787829906.279008:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c7015680 x1874670646876800/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787829906.279012:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 00000100:00000001:3.0:1787829906.279018:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787829906.279021:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829906.279022:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4299380480 00000100:00000001:3.0:1787829906.279024:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829906.279027:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 00000100:00080000:3.0:1787829906.279029:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:2.0:1787829906.279030:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787829906.279034:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829906.279037:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00080000:3.0:1787829906.279037:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1787829906.279041:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00100000:2.0:1787829906.279045:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829906/real 1787829906] req@ffff8d30c7016580 x1874670646876544/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829931 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787829906.279046:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787829906.279052:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000400:00000001:2.0:1787829906.279057:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00080000:3.0:1787829906.279057:0:40701:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4706 00000400:00000001:2.0:1787829906.279060:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1787829906.279064:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00080000:3.0:1787829906.279065:0:40701:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787829906.279067:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787829906.279069:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829906.279071:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829906.279072:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787829906.279075:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829906.279075:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:3.0:1787829906.279081:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000100:00100000:2.0:1787829906.279082:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c7016580 x1874670646876544/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829931 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:3.0:1787829906.279085:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1787829906.279089:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:2.0:1787829906.279091:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:3.0:1787829906.279093:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:2.0:1787829906.279095:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787829906.279099:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829906.279099:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829906.279102:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829906.279102:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:2.0:1787829906.279104:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829906.279107:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00080000:3.0:1787829906.279108:0:40701:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787829906.279109:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829906.279113:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787829906.279114:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 02000000:00000001:3.0:1787829906.279117:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00080000:2.0:1787829906.279120:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 02000000:00000001:3.0:1787829906.279120:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787829906.279122:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 02000000:00000001:3.0:1787829906.279124:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787829906.279127:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c7016580 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646876544:192.168.203.143@tcp:250:kworker.0 00000100:00000001:3.0:1787829906.279127:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:3.0:1787829906.279131:0:40701:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00000001:2.0:1787829906.279133:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000008:00000001:3.0:1787829906.279134:0:40701:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829906.279135:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787829906.279138:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 00000100:00000001:3.0:1787829906.279139:0:40701:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829906.279140:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1787829906.279143:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787829906.279146:0:2390:0:(genops.c:1217:class_import_put()) Process entered 02000000:00000001:3.0:1787829906.279146:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000020:00000001:2.0:1787829906.279147:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:3.0:1787829906.279150:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829906.279151:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787829906.279154:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829906.279154:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:2.0:1787829906.279158:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 02000000:00000001:3.0:1787829906.279158:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:2.0:1787829906.279162:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:3.0:1787829906.279163:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829906.279164:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829906.279171:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787829906.279171:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829906.279173:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829906.279176:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787829906.279177:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1787829906.279180:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00100000:3.0:1787829906.279180:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c7014f00 x1874670646876928/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787829906.279181:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787829906.279183:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829906.279187:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787829906.279189:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787829906.279194:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829906.279195:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00100000:2.0:1787829906.279198:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c7015a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646876672:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829906.279200:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829906.279202:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787829906.279204:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1787829906.279206:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829906.279208:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:3.0:1787829906.279208:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829906.279211:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829906.279216:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787829906.279217:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369542208 : -126234340009408 : ffff8d30c7015a40) 00000100:00000001:2.0:1787829906.279220:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1787829906.279229:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:2.0:1787829906.279232:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1787829906.279234:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787829906.279235:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829906.279236:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1787829906.279237:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829906.279241:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4299379222 00000100:00000001:2.0:1787829906.279243:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4299380480 00000100:00000001:2.0:1787829906.279246:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4299379222 00000100:00000001:2.0:1787829906.279248:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4299380480 00000100:00000001:2.0:1787829906.279250:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787829906.279252:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829906.279254:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1787829906.279258:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829906/real 1787829906] req@ffff8d30c7015a40 x1874670646876672/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829931 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1787829906.279266:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787829906.279267:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1787829906.279270:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1787829906.279271:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787829906.279273:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829906.279274:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787829906.279276:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787829906.279280:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c7015a40 x1874670646876672/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829931 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787829906.279287:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829906.279289:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787829906.279291:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829906.279293:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787829906.279294:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829906.279295:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787829906.279296:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787829906.279298:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829906.279300:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:2.0:1787829906.279301:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1787829906.279303:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c7015a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646876672:192.168.203.143@tcp:38:kworker.0 00000100:00000001:2.0:1787829906.279306:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1787829906.279307:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787829906.279308:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829906.279309:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787829906.279310:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787829906.279311:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787829906.279313:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c7015680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646876800:192.168.203.143@tcp:38:kworker.0 00000100:00000001:2.0:1787829906.279315:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787829906.279316:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1787829906.279317:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829906.279318:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1787829906.279320:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829906.279323:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787829906.279324:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369541248 : -126234340010368 : ffff8d30c7015680) 00000100:00000001:2.0:1787829906.279327:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1787829906.279334:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:2.0:1787829906.279337:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1787829906.279338:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787829906.279339:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829906.279340:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1787829906.279342:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829906.279344:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4299379222 00000100:00000001:2.0:1787829906.279345:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4299380480 00000100:00000001:2.0:1787829906.279347:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4299379222 00000100:00000001:2.0:1787829906.279348:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4299380480 00000100:00000001:2.0:1787829906.279350:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787829906.279351:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829906.279352:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1787829906.279354:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829906/real 1787829906] req@ffff8d30c7015680 x1874670646876800/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829931 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1787829906.279359:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787829906.279360:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1787829906.279362:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1787829906.279363:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787829906.279364:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829906.279366:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787829906.279367:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787829906.279370:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c7015680 x1874670646876800/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829931 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787829906.279377:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829906.279378:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787829906.279381:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829906.279382:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787829906.279383:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829906.279385:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787829906.279386:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787829906.279388:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829906.279391:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:2.0:1787829906.279394:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1787829906.279397:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c7015680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646876800:192.168.203.143@tcp:38:kworker.0 00000100:00000001:2.0:1787829906.279400:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829906.279402:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787829906.279404:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787829906.279405:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1787829906.279407:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787829906.279408:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:2.0:1787829906.279409:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:2.0:1787829906.279410:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787829906.279411:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1787829906.279412:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787829906.279413:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829906.279414:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787829906.279415:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787829906.279415:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1787829906.279416:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787829906.279417:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:2.0:1787829906.279417:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:2.0:1787829906.279418:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787829906.279419:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1787829906.279420:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787829906.279420:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829906.279421:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829906.279423:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1787829906.279424:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829906.279425:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787829906.279426:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1787829906.279427:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1787829906.279427:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787829906.279428:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829906.279429:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787829906.279431:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787829906.279432:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787829906.279435:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c7014f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646876928:192.168.203.143@tcp:8:kworker.0 00000100:00000001:2.0:1787829906.279438:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787829906.279440:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1787829906.279441:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829906.279443:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1787829906.279445:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829906.279451:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787829906.279452:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369539328 : -126234340012288 : ffff8d30c7014f00) 00000100:00000001:2.0:1787829906.279455:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1787829906.279467:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:2.0:1787829906.279471:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1787829906.279473:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787829906.279474:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829906.279476:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1787829906.279479:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829906.279482:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4299379235 00000100:00000001:2.0:1787829906.279484:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4299380480 00000100:00000001:2.0:1787829906.279486:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4299379235 00000100:00000001:2.0:1787829906.279487:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4299380480 00000100:00000001:2.0:1787829906.279489:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787829906.279491:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829906.279492:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1787829906.279496:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829906/real 1787829906] req@ffff8d30c7014f00 x1874670646876928/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829931 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1787829906.279504:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787829906.279505:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1787829906.279507:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1787829906.279509:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787829906.279510:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829906.279511:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787829906.279513:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787829906.279516:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c7014f00 x1874670646876928/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829931 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787829906.279522:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829906.279524:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787829906.279526:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829906.279528:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787829906.279529:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829906.279530:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787829906.279532:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787829906.279534:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829906.279538:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:2.0:1787829906.279540:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1787829906.279543:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c7014f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646876928:192.168.203.143@tcp:8:kworker.0 00000100:00000001:2.0:1787829906.279546:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829906.279547:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787829906.279548:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787829906.279549:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1787829906.279551:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787829906.279552:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:2.0:1787829906.279552:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:2.0:1787829906.279554:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787829906.279555:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1787829906.279556:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787829906.279557:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829906.279558:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829906.279560:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1787829906.279561:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829906.279562:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787829906.279563:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829907.174702:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829907.174707:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829907.174743:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829907.174746:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829907.174748:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829908.198708:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829908.198714:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829908.198764:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829908.198767:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829908.198769:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829909.222700:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829909.222706:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829909.222788:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829909.222792:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829909.222795:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829910.246712:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829910.246720:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829910.246816:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829910.246821:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829910.246826:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829911.270743:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829911.270751:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829911.270843:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829911.270848:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829911.270851:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787829911.398791:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829911.398805:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1787829911.398808:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787829911.398813:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787829911.398819:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787829911.398827:0:40701:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 4711 00000100:00080000:0.0:1787829911.398841:0:40701:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787829911.398845:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829911.398847:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787829911.398852:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787829911.398855:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787829911.398859:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:0.0:1787829911.398862:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:0.0:1787829911.398865:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829911.398866:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:0.0:1787829911.398871:0:40701:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787829911.398874:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829911.398879:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787829911.398883:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787829911.398887:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829911.398890:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1787829911.398893:0:40701:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829911.398908:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787829911.398910:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829911.398915:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787829911.398918:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787829911.398922:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829911.398936:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787829911.398948:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d25a40 x1874670646877056/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787829911.398960:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787829911.398965:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829911.398983:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787829911.398987:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829911.398990:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1787829911.398992:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787829911.398994:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787829911.398997:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787829911.398999:0:40701:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4711 00000100:00080000:0.0:1787829911.399004:0:40701:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787829911.399006:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829911.399008:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787829911.399010:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787829911.399012:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787829911.399015:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000020:00000001:0.0:1787829911.399017:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:0.0:1787829911.399019:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829911.399021:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:2.0:1787829911.399023:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00080000:0.0:1787829911.399026:0:40701:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787829911.399028:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1787829911.399032:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1787829911.399033:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829911.399035:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 02000000:00000001:0.0:1787829911.399037:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:2.0:1787829911.399038:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829911.399043:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787829911.399045:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787829911.399046:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787829911.399049:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829911.399051:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787829911.399055:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d25a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646877056:192.168.203.143@tcp:250:kworker.0 00000100:00000001:0.0:1787829911.399056:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:2.0:1787829911.399061:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000008:00100000:0.0:1787829911.399064:0:40701:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 02000000:00000001:2.0:1787829911.399066:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000008:00000001:0.0:1787829911.399070:0:40701:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829911.399070:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829911.399075:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:0.0:1787829911.399076:0:40701:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829911.399080:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829911.399083:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787829911.399085:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829911.399088:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787829911.399090:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787829911.399091:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829911.399097:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829911.399098:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787829911.399100:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601343040 : -126234108208576 : ffff8d30d4d25a40) 00000100:00100000:0.0:1787829911.399105:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d270c0 x1874670646877184/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787829911.399106:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:0.0:1787829911.399116:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787829911.399119:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829911.399123:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787829911.399126:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829911.399129:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1787829911.399131:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787829911.399133:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787829911.399136:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787829911.399139:0:40701:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4711 00000100:00080000:0.0:1787829911.399143:0:40701:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787829911.399146:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829911.399149:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000800:00000100:2.0:1787829911.399149:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000020:00000001:0.0:1787829911.399154:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787829911.399155:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787829911.399158:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000020:00000001:0.0:1787829911.399160:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:2.0:1787829911.399163:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1787829911.399164:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829911.399167:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:2.0:1787829911.399168:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787829911.399171:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829911.399173:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00080000:0.0:1787829911.399176:0:40701:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:2.0:1787829911.399178:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829911.399180:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829911.399182:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4299385600 02000000:00000001:0.0:1787829911.399184:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787829911.399188:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787829911.399191:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829911.399193:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1787829911.399194:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1787829911.399198:0:40701:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00000001:2.0:1787829911.399200:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4299385600 00000008:00000001:0.0:1787829911.399203:0:40701:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829911.399205:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1787829911.399207:0:40701:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829911.399209:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787829911.399211:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829911.399214:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:2.0:1787829911.399214:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 02000000:00000001:0.0:1787829911.399218:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787829911.399219:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829911/real 1787829911] req@ffff8d30d4d25a40 x1874670646877056/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829936 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787829911.399222:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787829911.399226:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787829911.399229:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:2.0:1787829911.399231:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787829911.399233:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787829911.399236:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829911.399236:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1787829911.399240:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787829911.399242:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829911.399245:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00100000:0.0:1787829911.399246:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d24000 x1874670646877312/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787829911.399247:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787829911.399254:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d25a40 x1874670646877056/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829936 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787829911.399267:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787829911.399271:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829911.399276:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787829911.399282:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:2.0:1787829911.399283:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:0.0:1787829911.399289:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:2.0:1787829911.399289:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787829911.399293:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:2.0:1787829911.399295:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:0.0:1787829911.399297:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:2.0:1787829911.399298:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787829911.399301:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829911.399302:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:2.0:1787829911.399304:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00080000:0.0:1787829911.399309:0:40701:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4711 00000100:00000001:2.0:1787829911.399309:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787829911.399316:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829911.399317:0:40701:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787829911.399322:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00080000:2.0:1787829911.399322:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:2.0:1787829911.399325:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000001:0.0:1787829911.399326:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00100000:2.0:1787829911.399329:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d25a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646877056:192.168.203.143@tcp:250:kworker.0 00000020:00000001:0.0:1787829911.399332:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787829911.399335:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:2.0:1787829911.399335:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1787829911.399338:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:2.0:1787829911.399339:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787829911.399341:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 00000020:00000001:0.0:1787829911.399344:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 02000000:00000001:2.0:1787829911.399345:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1787829911.399349:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00000001:0.0:1787829911.399350:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000020:00000001:2.0:1787829911.399352:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000100:00000001:0.0:1787829911.399353:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:2.0:1787829911.399354:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:2.0:1787829911.399359:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 00000100:00080000:0.0:1787829911.399361:0:40701:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 02000000:00000001:2.0:1787829911.399362:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1787829911.399366:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787829911.399367:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829911.399368:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829911.399371:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:0.0:1787829911.399372:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787829911.399377:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787829911.399379:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 02000000:00000001:0.0:1787829911.399380:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829911.399382:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829911.399384:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:2.0:1787829911.399384:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787829911.399387:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000008:00100000:0.0:1787829911.399389:0:40701:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00000001:2.0:1787829911.399389:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1787829911.399391:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787829911.399393:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:0.0:1787829911.399395:0:40701:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829911.399396:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787829911.399398:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787829911.399401:0:40701:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829911.399401:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787829911.399406:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d270c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646877184:192.168.203.143@tcp:38:kworker.0 00000100:00000001:2.0:1787829911.399408:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787829911.399410:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:2.0:1787829911.399410:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1787829911.399413:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829911.399414:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829911.399414:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1787829911.399417:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829911.399419:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787829911.399421:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787829911.399424:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829911.399424:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787829911.399426:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601348800 : -126234108202816 : ffff8d30d4d270c0) 00000100:00000001:2.0:1787829911.399431:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:0.0:1787829911.399433:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787829911.399441:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d24f00 x1874670646877440/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1787829911.399444:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:2.0:1787829911.399449:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1787829911.399451:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787829911.399453:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829911.399456:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787829911.399456:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1787829911.399459:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829911.399461:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829911.399464:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4299384342 00000100:00000001:0.0:1787829911.399467:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829911.399467:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4299385600 00000100:00000001:2.0:1787829911.399472:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4299384342 00000100:00000001:2.0:1787829911.399474:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4299385600 00000100:00000001:2.0:1787829911.399476:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787829911.399477:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829911.399478:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1787829911.399482:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829911/real 1787829911] req@ffff8d30d4d270c0 x1874670646877184/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829936 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1787829911.399487:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787829911.399488:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1787829911.399491:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1787829911.399492:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787829911.399493:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829911.399494:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787829911.399495:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787829911.399498:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d270c0 x1874670646877184/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829936 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787829911.399507:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829911.399509:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787829911.399512:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829911.399513:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787829911.399514:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829911.399516:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787829911.399518:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787829911.399520:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829911.399523:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:2.0:1787829911.399525:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1787829911.399528:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d270c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646877184:192.168.203.143@tcp:38:kworker.0 00000100:00000001:2.0:1787829911.399532:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1787829911.399533:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787829911.399534:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829911.399536:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787829911.399538:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787829911.399541:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787829911.399543:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d24000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646877312:192.168.203.143@tcp:38:kworker.0 00000100:00000001:2.0:1787829911.399546:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787829911.399548:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1787829911.399550:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829911.399551:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1787829911.399555:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829911.399558:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787829911.399560:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601336320 : -126234108215296 : ffff8d30d4d24000) 00000100:00000001:2.0:1787829911.399563:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1787829911.399572:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:2.0:1787829911.399575:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1787829911.399576:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787829911.399577:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829911.399579:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1787829911.399580:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829911.399583:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4299384342 00000100:00000001:2.0:1787829911.399584:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4299385600 00000100:00000001:2.0:1787829911.399586:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4299384342 00000100:00000001:2.0:1787829911.399587:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4299385600 00000100:00000001:2.0:1787829911.399588:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787829911.399589:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829911.399590:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1787829911.399593:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829911/real 1787829911] req@ffff8d30d4d24000 x1874670646877312/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829936 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1787829911.399597:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787829911.399597:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1787829911.399599:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1787829911.399626:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787829911.399627:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829911.399628:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787829911.399629:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787829911.399632:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d24000 x1874670646877312/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829936 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787829911.399636:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829911.399638:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787829911.399639:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829911.399640:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787829911.399641:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829911.399641:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787829911.399642:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787829911.399644:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829911.399645:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:2.0:1787829911.399646:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1787829911.399649:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d24000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646877312:192.168.203.143@tcp:38:kworker.0 00000100:00000001:2.0:1787829911.399651:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829911.399652:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787829911.399653:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787829911.399654:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1787829911.399655:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787829911.399656:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:2.0:1787829911.399657:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:2.0:1787829911.399658:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787829911.399659:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1787829911.399662:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787829911.399663:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829911.399664:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787829911.399664:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787829911.399665:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1787829911.399666:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787829911.399667:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:2.0:1787829911.399667:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:2.0:1787829911.399668:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787829911.399669:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1787829911.399670:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787829911.399671:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829911.399672:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829911.399674:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1787829911.399674:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829911.399676:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787829911.399676:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1787829911.399677:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1787829911.399678:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787829911.399679:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829911.399680:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787829911.399681:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787829911.399682:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787829911.399684:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d24f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646877440:192.168.203.143@tcp:8:kworker.0 00000100:00000001:2.0:1787829911.399686:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787829911.399687:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1787829911.399688:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829911.399689:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1787829911.399690:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829911.399694:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787829911.399695:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601340160 : -126234108211456 : ffff8d30d4d24f00) 00000100:00000001:2.0:1787829911.399697:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1787829911.399704:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:2.0:1787829911.399706:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1787829911.399707:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787829911.399708:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829911.399709:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1787829911.399711:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829911.399713:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4299384356 00000100:00000001:2.0:1787829911.399714:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4299385600 00000100:00000001:2.0:1787829911.399716:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4299384356 00000100:00000001:2.0:1787829911.399717:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4299385600 00000100:00000001:2.0:1787829911.399718:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787829911.399720:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829911.399721:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1787829911.399725:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829911/real 1787829911] req@ffff8d30d4d24f00 x1874670646877440/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829936 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1787829911.399732:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787829911.399733:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1787829911.399735:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1787829911.399737:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787829911.399738:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829911.399740:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787829911.399741:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787829911.399744:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d24f00 x1874670646877440/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829936 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787829911.399752:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829911.399754:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787829911.399756:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829911.399758:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787829911.399778:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829911.399780:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787829911.399782:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787829911.399784:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829911.399789:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:2.0:1787829911.399790:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1787829911.399792:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d24f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646877440:192.168.203.143@tcp:8:kworker.0 00000100:00000001:2.0:1787829911.399795:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829911.399796:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787829911.399797:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787829911.399798:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1787829911.399800:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787829911.399801:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:2.0:1787829911.399801:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:2.0:1787829911.399803:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787829911.399804:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1787829911.399805:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787829911.399806:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829911.399807:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829911.399809:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1787829911.399810:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829911.399811:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787829911.399813:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829912.294711:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829912.294721:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829912.294834:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829912.294840:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829912.294845:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829913.318735:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829913.318739:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829913.318824:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829913.318827:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829913.318831:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829914.342720:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829914.342726:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829914.342775:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829914.342779:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829914.342782:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829915.366687:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829915.366692:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829915.366761:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829915.366765:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829915.366768:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829916.390710:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829916.390716:0:815:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1787829916.390719:0:815:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1787829916.390722:0:815:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829916.390736:0:815:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1787829916.390738:0:815:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829916.390740:0:815:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829916.390743:0:815:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829916.390836:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829916.390842:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829916.390847:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787829916.518692:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829916.518700:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1787829916.518703:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787829916.518706:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787829916.518708:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787829916.518714:0:40701:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 4717 00000100:00080000:0.0:1787829916.518722:0:40701:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787829916.518735:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829916.518736:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787829916.518740:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787829916.518741:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787829916.518744:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:0.0:1787829916.518745:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:0.0:1787829916.518746:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829916.518747:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:0.0:1787829916.518750:0:40701:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787829916.518752:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829916.518754:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787829916.518756:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787829916.518758:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829916.518759:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1787829916.518761:0:40701:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829916.518771:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787829916.518772:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829916.518775:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787829916.518778:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787829916.518781:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829916.518793:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787829916.518802:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d24f00 x1874670646877568/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787829916.518811:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787829916.518815:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829916.518826:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787829916.518828:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829916.518831:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1787829916.518832:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787829916.518833:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787829916.518835:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787829916.518837:0:40701:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4717 00000100:00080000:0.0:1787829916.518840:0:40701:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787829916.518841:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829916.518842:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787829916.518844:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787829916.518845:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787829916.518847:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000020:00000001:0.0:1787829916.518848:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:0.0:1787829916.518850:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829916.518850:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:0.0:1787829916.518853:0:40701:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787829916.518855:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829916.518856:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:0.0:1787829916.518857:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:2.0:1787829916.518861:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 02000000:00000001:0.0:1787829916.518864:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787829916.518864:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1787829916.518868:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 02000000:00000001:0.0:1787829916.518869:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829916.518871:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829916.518873:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 02000000:00000001:2.0:1787829916.518877:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000008:00100000:0.0:1787829916.518879:0:40701:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 02000000:00000001:2.0:1787829916.518881:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000008:00000001:0.0:1787829916.518885:0:40701:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829916.518886:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829916.518889:0:40701:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787829916.518894:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d24f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646877568:192.168.203.143@tcp:250:kworker.0 02000000:00000001:0.0:1787829916.518896:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787829916.518899:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829916.518902:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:2.0:1787829916.518902:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787829916.518905:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787829916.518907:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787829916.518908:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829916.518912:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829916.518915:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829916.518916:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1787829916.518923:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787829916.518924:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d24000 x1874670646877696/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787829916.518933:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787829916.518936:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829916.518936:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787829916.518939:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601340160 : -126234108211456 : ffff8d30d4d24f00) 00000100:00000001:0.0:1787829916.518940:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829916.518943:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00080000:0.0:1787829916.518945:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829916.518949:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1787829916.518951:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787829916.518952:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787829916.518955:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787829916.518957:0:40701:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4717 00000100:00080000:0.0:1787829916.518961:0:40701:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787829916.518963:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829916.518965:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787829916.518967:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787829916.518969:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787829916.518970:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000020:00000001:0.0:1787829916.518972:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:0.0:1787829916.518974:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829916.518975:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:0.0:1787829916.518980:0:40701:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000800:00000100:2.0:1787829916.518982:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:0.0:1787829916.518985:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829916.518988:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787829916.518991:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787829916.518993:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829916.518996:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:2.0:1787829916.518996:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000008:00100000:0.0:1787829916.518998:0:40701:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1787829916.519002:0:40701:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829916.519002:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787829916.519006:0:40701:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829916.519006:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829916.519011:0:2390:0:(events.c:68:request_out_callback()) Process leaving 02000000:00000001:0.0:1787829916.519013:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787829916.519017:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829916.519018:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829916.519019:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787829916.519022:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787829916.519025:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829916.519025:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4299390720 00000100:00000001:0.0:1787829916.519030:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829916.519036:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 00000100:00100000:0.0:1787829916.519037:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d270c0 x1874670646877824/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787829916.519044:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4299390720 00000100:00000001:0.0:1787829916.519046:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787829916.519048:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1787829916.519050:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829916.519052:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787829916.519054:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829916.519055:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829916.519060:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00080000:0.0:1787829916.519061:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1787829916.519066:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829916/real 1787829916] req@ffff8d30d4d24f00 x1874670646877568/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829941 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829916.519067:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1787829916.519071:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787829916.519074:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787829916.519080:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787829916.519086:0:40701:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4717 00000400:00000001:2.0:1787829916.519086:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787829916.519089:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00080000:0.0:1787829916.519092:0:40701:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787829916.519094:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787829916.519096:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829916.519098:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787829916.519099:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:2.0:1787829916.519101:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1787829916.519104:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:2.0:1787829916.519105:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000020:00000001:0.0:1787829916.519106:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787829916.519109:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:2.0:1787829916.519109:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1787829916.519114:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:0.0:1787829916.519116:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829916.519117:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00100000:2.0:1787829916.519118:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d24f00 x1874670646877568/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829941 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829916.519122:0:40701:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787829916.519127:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829916.519132:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:2.0:1787829916.519133:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:0.0:1787829916.519136:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787829916.519136:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 02000000:00000001:0.0:1787829916.519142:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829916.519142:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829916.519145:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787829916.519148:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:2.0:1787829916.519148:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829916.519151:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000008:00100000:0.0:1787829916.519152:0:40701:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00000001:2.0:1787829916.519153:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787829916.519157:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000008:00000001:0.0:1787829916.519159:0:40701:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787829916.519161:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787829916.519164:0:40701:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829916.519164:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1787829916.519170:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d24f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646877568:192.168.203.143@tcp:250:kworker.0 02000000:00000001:0.0:1787829916.519172:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:2.0:1787829916.519173:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:0.0:1787829916.519176:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829916.519176:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787829916.519178:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 00000100:00000001:0.0:1787829916.519181:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:2.0:1787829916.519181:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787829916.519185:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:2.0:1787829916.519185:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787829916.519188:0:2390:0:(genops.c:1217:class_import_put()) Process entered 02000000:00000001:0.0:1787829916.519190:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:2.0:1787829916.519190:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:2.0:1787829916.519194:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787829916.519195:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787829916.519197:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829916.519199:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787829916.519202:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829916.519203:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829916.519209:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1787829916.519211:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829916.519212:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787829916.519214:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00100000:0.0:1787829916.519215:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d25a40 x1874670646877952/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787829916.519215:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1787829916.519218:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787829916.519220:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829916.519223:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787829916.519225:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787829916.519228:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 02000000:00000001:2.0:1787829916.519229:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787829916.519234:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d24000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646877696:192.168.203.143@tcp:38:kworker.0 00000100:00000001:2.0:1787829916.519236:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00000001:0.0:1787829916.519237:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829916.519238:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1787829916.519240:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829916.519242:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00000001:0.0:1787829916.519243:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829916.519245:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829916.519251:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787829916.519253:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601336320 : -126234108215296 : ffff8d30d4d24000) 00000100:00000001:2.0:1787829916.519256:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1787829916.519265:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:2.0:1787829916.519269:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1787829916.519270:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787829916.519271:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829916.519272:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1787829916.519274:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829916.519277:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4299389462 00000100:00000001:2.0:1787829916.519280:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4299390720 00000100:00000001:2.0:1787829916.519283:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4299389462 00000100:00000001:2.0:1787829916.519286:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4299390720 00000100:00000001:2.0:1787829916.519288:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787829916.519290:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829916.519292:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1787829916.519296:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829916/real 1787829916] req@ffff8d30d4d24000 x1874670646877696/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829941 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1787829916.519311:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787829916.519313:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1787829916.519316:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1787829916.519318:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787829916.519319:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829916.519321:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787829916.519323:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787829916.519327:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d24000 x1874670646877696/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829941 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787829916.519333:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829916.519335:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787829916.519343:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829916.519345:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787829916.519347:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829916.519349:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787829916.519350:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787829916.519353:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829916.519358:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:2.0:1787829916.519360:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1787829916.519364:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d24000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646877696:192.168.203.143@tcp:38:kworker.0 00000100:00000001:2.0:1787829916.519368:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1787829916.519370:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787829916.519371:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829916.519375:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787829916.519377:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787829916.519379:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787829916.519382:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d270c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646877824:192.168.203.143@tcp:38:kworker.0 00000100:00000001:2.0:1787829916.519384:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787829916.519386:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1787829916.519387:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829916.519389:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1787829916.519391:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829916.519397:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787829916.519398:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601348800 : -126234108202816 : ffff8d30d4d270c0) 00000100:00000001:2.0:1787829916.519401:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1787829916.519411:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:2.0:1787829916.519416:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1787829916.519417:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787829916.519418:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829916.519420:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1787829916.519422:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829916.519426:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4299389462 00000100:00000001:2.0:1787829916.519428:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4299390720 00000100:00000001:2.0:1787829916.519430:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4299389462 00000100:00000001:2.0:1787829916.519432:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4299390720 00000100:00000001:2.0:1787829916.519434:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787829916.519435:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829916.519436:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1787829916.519440:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829916/real 1787829916] req@ffff8d30d4d270c0 x1874670646877824/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829941 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1787829916.519446:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787829916.519447:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1787829916.519449:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1787829916.519450:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787829916.519452:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829916.519453:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787829916.519454:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787829916.519457:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d270c0 x1874670646877824/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829941 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787829916.519462:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829916.519464:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787829916.519466:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829916.519467:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787829916.519468:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829916.519469:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787829916.519470:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787829916.519472:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829916.519475:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:2.0:1787829916.519476:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1787829916.519479:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d270c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646877824:192.168.203.143@tcp:38:kworker.0 00000100:00000001:2.0:1787829916.519482:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829916.519484:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787829916.519485:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787829916.519486:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1787829916.519488:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787829916.519489:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:2.0:1787829916.519490:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:2.0:1787829916.519492:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787829916.519493:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1787829916.519495:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787829916.519496:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829916.519497:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787829916.519498:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787829916.519499:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1787829916.519500:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787829916.519501:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:2.0:1787829916.519502:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:2.0:1787829916.519503:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787829916.519504:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1787829916.519507:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787829916.519508:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829916.519509:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829916.519512:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1787829916.519514:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829916.519516:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787829916.519517:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1787829916.519518:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1787829916.519519:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787829916.519521:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829916.519523:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787829916.519524:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787829916.519526:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787829916.519529:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d25a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646877952:192.168.203.143@tcp:8:kworker.0 00000100:00000001:2.0:1787829916.519532:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787829916.519534:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1787829916.519535:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829916.519537:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1787829916.519539:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829916.519544:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787829916.519546:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601343040 : -126234108208576 : ffff8d30d4d25a40) 00000100:00000001:2.0:1787829916.519549:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1787829916.519560:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:2.0:1787829916.519565:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1787829916.519567:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787829916.519568:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829916.519569:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1787829916.519572:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829916.519575:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4299389475 00000100:00000001:2.0:1787829916.519578:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4299390720 00000100:00000001:2.0:1787829916.519580:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4299389475 00000100:00000001:2.0:1787829916.519581:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4299390720 00000100:00000001:2.0:1787829916.519583:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787829916.519584:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829916.519585:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1787829916.519588:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829916/real 1787829916] req@ffff8d30d4d25a40 x1874670646877952/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829941 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1787829916.519592:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787829916.519593:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1787829916.519595:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1787829916.519596:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787829916.519597:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829916.519598:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787829916.519599:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787829916.519622:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d25a40 x1874670646877952/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829941 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787829916.519627:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829916.519628:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787829916.519630:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829916.519631:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787829916.519631:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829916.519632:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787829916.519633:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787829916.519635:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829916.519636:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:2.0:1787829916.519637:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1787829916.519639:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d25a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646877952:192.168.203.143@tcp:8:kworker.0 00000100:00000001:2.0:1787829916.519642:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829916.519643:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787829916.519644:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787829916.519644:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1787829916.519646:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787829916.519646:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:2.0:1787829916.519647:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:2.0:1787829916.519648:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787829916.519649:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1787829916.519650:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787829916.519651:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829916.519652:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829916.519654:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1787829916.519655:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829916.519656:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787829916.519657:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829917.414680:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829917.414684:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829917.414766:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829917.414769:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829917.414772:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829918.438673:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829918.438680:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829918.438750:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829918.438754:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829918.438757:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829919.462663:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829919.462668:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829919.462723:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829919.462726:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829919.462729:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829920.486732:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829920.486737:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829920.486793:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829920.486797:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829920.486800:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829921.510657:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829921.510662:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829921.510722:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829921.510725:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829921.510727:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787829921.638706:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829921.638713:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1787829921.638715:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787829921.638718:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787829921.638733:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787829921.638739:0:40701:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 4722 00000100:00080000:0.0:1787829921.638747:0:40701:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787829921.638749:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829921.638751:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787829921.638754:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787829921.638756:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787829921.638758:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:0.0:1787829921.638760:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:0.0:1787829921.638762:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829921.638763:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:0.0:1787829921.638766:0:40701:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787829921.638768:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829921.638770:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787829921.638772:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787829921.638775:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829921.638776:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1787829921.638778:0:40701:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829921.638787:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787829921.638788:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829921.638790:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787829921.638792:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787829921.638795:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829921.638803:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787829921.638810:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d252c0 x1874670646878080/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787829921.638816:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787829921.638819:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829921.638827:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787829921.638830:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829921.638842:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1787829921.638844:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00000001:2.0:1787829921.638844:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00080000:0.0:1787829921.638848:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787829921.638850:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:2.0:1787829921.638850:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00080000:0.0:1787829921.638854:0:40701:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4722 00000100:00000001:2.0:1787829921.638855:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00080000:0.0:1787829921.638857:0:40701:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787829921.638858:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787829921.638860:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829921.638862:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:2.0:1787829921.638863:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787829921.638865:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787829921.638867:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787829921.638869:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 02000000:00000001:2.0:1787829921.638869:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000020:00000001:0.0:1787829921.638870:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:0.0:1787829921.638873:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 02000000:00000001:2.0:1787829921.638874:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787829921.638875:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:0.0:1787829921.638880:0:40701:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 02000000:00000001:2.0:1787829921.638881:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829921.638882:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787829921.638889:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d252c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646878080:192.168.203.143@tcp:250:kworker.0 02000000:00000001:0.0:1787829921.638893:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:2.0:1787829921.638893:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787829921.638896:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787829921.638899:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787829921.638899:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829921.638901:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1787829921.638904:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829921.638907:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 02000000:00000001:2.0:1787829921.638907:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000008:00100000:0.0:1787829921.638912:0:40701:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1787829921.638915:0:40701:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829921.638916:0:40701:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829921.638920:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:2.0:1787829921.638920:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 02000000:00000001:0.0:1787829921.638921:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829921.638922:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601341120 : -126234108210496 : ffff8d30d4d252c0) 00000100:00000001:0.0:1787829921.638924:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787829921.638927:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:2.0:1787829921.638927:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 02000000:00000001:0.0:1787829921.638931:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829921.638935:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787829921.638939:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d25680 x1874670646878208/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787829921.638945:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787829921.638946:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829921.638947:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787829921.638949:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829921.638952:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1787829921.638953:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787829921.638954:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787829921.638956:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787829921.638957:0:40701:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4722 00000100:00080000:0.0:1787829921.638960:0:40701:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787829921.638961:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829921.638962:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787829921.638963:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787829921.638964:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787829921.638965:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000020:00000001:0.0:1787829921.638969:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000800:00000100:2.0:1787829921.638969:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:0.0:1787829921.638973:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829921.638974:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:0.0:1787829921.638977:0:40701:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787829921.638979:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829921.638980:0:2390:0:(events.c:34:request_out_callback()) Process entered 02000000:00000001:0.0:1787829921.638981:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787829921.638984:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787829921.638985:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 02000000:00000001:0.0:1787829921.638986:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829921.638988:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:2.0:1787829921.638988:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829921.638991:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000008:00100000:0.0:1787829921.638992:0:40701:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1787829921.638995:0:40701:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829921.638997:0:40701:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829921.638997:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829921.639002:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:2.0:1787829921.639002:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4299395840 02000000:00000001:0.0:1787829921.639004:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829921.639007:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787829921.639008:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787829921.639009:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829921.639012:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1787829921.639013:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787829921.639019:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d27c00 x1874670646878336/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787829921.639020:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4299395840 00000100:00000001:2.0:1787829921.639026:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1787829921.639027:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787829921.639029:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787829921.639030:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829921.639033:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829921.639034:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787829921.639039:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:2.0:1787829921.639039:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00080000:0.0:1787829921.639043:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1787829921.639045:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00100000:2.0:1787829921.639046:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829921/real 1787829921] req@ffff8d30d4d252c0 x1874670646878080/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829946 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829921.639047:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787829921.639052:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787829921.639056:0:40701:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4722 00000100:00080000:0.0:1787829921.639061:0:40701:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000400:00000001:2.0:1787829921.639061:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00000001:0.0:1787829921.639064:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000001:2.0:1787829921.639064:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787829921.639067:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:2.0:1787829921.639068:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000020:00000001:0.0:1787829921.639069:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787829921.639071:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:2.0:1787829921.639071:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000020:00000001:0.0:1787829921.639073:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:2.0:1787829921.639074:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1787829921.639078:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:2.0:1787829921.639078:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787829921.639081:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:2.0:1787829921.639081:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829921.639084:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00100000:2.0:1787829921.639086:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d252c0 x1874670646878080/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829946 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829921.639089:0:40701:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787829921.639093:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829921.639097:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787829921.639100:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787829921.639103:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829921.639103:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829921.639109:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1787829921.639110:0:40701:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00000001:2.0:1787829921.639111:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000008:00000001:0.0:1787829921.639112:0:40701:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829921.639116:0:40701:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829921.639118:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:0.0:1787829921.639120:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787829921.639122:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829921.639122:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787829921.639126:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:2.0:1787829921.639126:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:0.0:1787829921.639128:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787829921.639131:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829921.639131:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787829921.639135:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787829921.639139:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:2.0:1787829921.639140:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:0.0:1787829921.639146:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d26940 x1874670646878464/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1787829921.639148:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:2.0:1787829921.639153:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000001:0.0:1787829921.639157:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787829921.639160:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787829921.639161:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d252c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646878080:192.168.203.143@tcp:250:kworker.0 00000100:00000001:0.0:1787829921.639162:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829921.639171:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829921.639174:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787829921.639176:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787829921.639179:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1787829921.639182:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787829921.639184:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:2.0:1787829921.639186:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:2.0:1787829921.639190:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787829921.639192:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1787829921.639195:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787829921.639197:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829921.639199:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829921.639205:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1787829921.639207:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829921.639209:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787829921.639210:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1787829921.639212:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1787829921.639213:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787829921.639214:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829921.639216:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787829921.639218:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787829921.639221:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787829921.639224:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d25680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646878208:192.168.203.143@tcp:38:kworker.0 00000100:00000001:2.0:1787829921.639227:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787829921.639229:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1787829921.639231:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829921.639233:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1787829921.639235:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829921.639241:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787829921.639242:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601342080 : -126234108209536 : ffff8d30d4d25680) 00000100:00000001:2.0:1787829921.639245:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1787829921.639255:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:2.0:1787829921.639258:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1787829921.639259:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787829921.639260:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829921.639262:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1787829921.639263:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829921.639266:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4299394582 00000100:00000001:2.0:1787829921.639268:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4299395840 00000100:00000001:2.0:1787829921.639270:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4299394582 00000100:00000001:2.0:1787829921.639271:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4299395840 00000100:00000001:2.0:1787829921.639272:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787829921.639273:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829921.639274:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1787829921.639277:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829921/real 1787829921] req@ffff8d30d4d25680 x1874670646878208/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829946 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1787829921.639283:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787829921.639283:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1787829921.639285:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1787829921.639286:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787829921.639287:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829921.639288:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787829921.639288:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787829921.639291:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d25680 x1874670646878208/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829946 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787829921.639295:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829921.639296:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787829921.639297:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829921.639298:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787829921.639299:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829921.639299:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787829921.639301:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787829921.639302:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829921.639304:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:2.0:1787829921.639305:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1787829921.639307:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d25680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646878208:192.168.203.143@tcp:38:kworker.0 00000100:00000001:2.0:1787829921.639308:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1787829921.639309:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787829921.639310:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829921.639311:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787829921.639313:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787829921.639321:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787829921.639323:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d27c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646878336:192.168.203.143@tcp:38:kworker.0 00000100:00000001:2.0:1787829921.639325:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787829921.639326:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1787829921.639327:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829921.639328:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1787829921.639330:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829921.639333:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787829921.639333:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601351680 : -126234108199936 : ffff8d30d4d27c00) 00000100:00000001:2.0:1787829921.639335:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1787829921.639342:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:2.0:1787829921.639344:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1787829921.639345:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787829921.639346:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829921.639347:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1787829921.639348:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829921.639350:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4299394582 00000100:00000001:2.0:1787829921.639352:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4299395840 00000100:00000001:2.0:1787829921.639353:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4299394582 00000100:00000001:2.0:1787829921.639354:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4299395840 00000100:00000001:2.0:1787829921.639355:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787829921.639362:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829921.639363:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1787829921.639365:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829921/real 1787829921] req@ffff8d30d4d27c00 x1874670646878336/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829946 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1787829921.639370:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787829921.639371:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1787829921.639372:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1787829921.639373:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787829921.639374:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829921.639375:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787829921.639376:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787829921.639378:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d27c00 x1874670646878336/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829946 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787829921.639382:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829921.639383:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787829921.639385:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829921.639386:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787829921.639386:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829921.639387:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787829921.639388:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787829921.639390:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829921.639391:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:2.0:1787829921.639392:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1787829921.639394:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d27c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646878336:192.168.203.143@tcp:38:kworker.0 00000100:00000001:2.0:1787829921.639396:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1787829921.639396:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787829921.639397:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829921.639398:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787829921.639399:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787829921.639400:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787829921.639402:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d26940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646878464:192.168.203.143@tcp:8:kworker.0 00000100:00000001:2.0:1787829921.639403:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787829921.639404:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1787829921.639405:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829921.639406:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1787829921.639408:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829921.639411:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787829921.639412:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601346880 : -126234108204736 : ffff8d30d4d26940) 00000100:00000001:2.0:1787829921.639413:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1787829921.639419:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:2.0:1787829921.639421:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1787829921.639422:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787829921.639423:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829921.639424:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1787829921.639425:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829921.639427:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4299394595 00000100:00000001:2.0:1787829921.639428:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4299395840 00000100:00000001:2.0:1787829921.639430:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4299394595 00000100:00000001:2.0:1787829921.639431:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4299395840 00000100:00000001:2.0:1787829921.639433:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787829921.639434:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829921.639435:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1787829921.639437:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829921/real 1787829921] req@ffff8d30d4d26940 x1874670646878464/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829946 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1787829921.639441:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787829921.639441:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1787829921.639442:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1787829921.639443:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787829921.639444:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829921.639444:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787829921.639445:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787829921.639447:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d26940 x1874670646878464/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829946 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787829921.639451:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829921.639452:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787829921.639453:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829921.639454:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787829921.639455:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829921.639456:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787829921.639457:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787829921.639458:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829921.639459:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:2.0:1787829921.639460:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1787829921.639462:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d26940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646878464:192.168.203.143@tcp:8:kworker.0 00000100:00000001:2.0:1787829921.639464:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829921.639465:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787829921.639466:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787829921.639467:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1787829921.639468:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787829921.639469:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:2.0:1787829921.639470:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:2.0:1787829921.639471:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787829921.639472:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1787829921.639473:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787829921.639474:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829921.639475:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787829921.639476:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787829921.639477:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1787829921.639477:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787829921.639478:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:2.0:1787829921.639479:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:2.0:1787829921.639479:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787829921.639480:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1787829921.639482:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787829921.639482:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829921.639483:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787829921.639484:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787829921.639484:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1787829921.639485:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787829921.639486:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:2.0:1787829921.639486:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:2.0:1787829921.639487:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787829921.639488:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1787829921.639490:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787829921.639490:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829921.639492:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829921.639494:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1787829921.639495:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829921.639496:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787829921.639497:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829922.534703:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829922.534707:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829922.534757:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829922.534760:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829922.534763:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829923.558712:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829923.558718:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829923.558776:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829923.558779:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829923.558782:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829924.582719:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829924.582724:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829924.582770:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829924.582772:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829924.582774:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829925.606704:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829925.606709:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829925.606789:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829925.606795:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829925.606799:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829926.630715:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829926.630723:0:815:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1787829926.630725:0:815:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1787829926.630728:0:815:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829926.630733:0:815:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1787829926.630735:0:815:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829926.630737:0:815:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829926.630739:0:815:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829926.630789:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829926.630792:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829926.630794:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787829926.758705:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829926.758714:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1787829926.758717:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787829926.758721:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787829926.758724:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787829926.758730:0:40701:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 4727 00000100:00080000:0.0:1787829926.758738:0:40701:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787829926.758740:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829926.758742:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787829926.758744:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787829926.758746:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787829926.758748:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:0.0:1787829926.758749:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:0.0:1787829926.758751:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829926.758752:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:0.0:1787829926.758755:0:40701:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787829926.758757:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829926.758759:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787829926.758761:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787829926.758763:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829926.758764:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1787829926.758766:0:40701:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829926.758774:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787829926.758775:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829926.758777:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787829926.758779:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787829926.758782:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829926.758791:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787829926.758797:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d26940 x1874670646878592/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787829926.758804:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787829926.758807:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829926.758818:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787829926.758820:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829926.758822:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1787829926.758824:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787829926.758824:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787829926.758826:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787829926.758827:0:40701:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4727 00000100:00080000:0.0:1787829926.758829:0:40701:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787829926.758830:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829926.758831:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787829926.758832:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787829926.758833:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787829926.758835:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000020:00000001:0.0:1787829926.758836:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:0.0:1787829926.758837:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829926.758838:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:0.0:1787829926.758840:0:40701:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787829926.758842:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829926.758843:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787829926.758852:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787829926.758854:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829926.758856:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1787829926.758862:0:40701:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1787829926.758866:0:40701:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829926.758868:0:40701:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829926.758869:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 02000000:00000001:0.0:1787829926.758872:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00000001:2.0:1787829926.758875:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 02000000:00000001:0.0:1787829926.758876:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829926.758880:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 00000100:00000001:2.0:1787829926.758880:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 02000000:00000001:0.0:1787829926.758885:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:2.0:1787829926.758885:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 02000000:00000001:0.0:1787829926.758887:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829926.758889:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829926.758895:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829926.758895:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787829926.758900:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00100000:0.0:1787829926.758904:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d27c00 x1874670646878720/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 02000000:00000001:2.0:1787829926.758905:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829926.758915:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00100000:2.0:1787829926.758915:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d26940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646878592:192.168.203.143@tcp:250:kworker.0 00000100:00000001:0.0:1787829926.758919:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829926.758923:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829926.758925:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 00000100:00080000:0.0:1787829926.758927:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829926.758930:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 02000000:00000001:2.0:1787829926.758930:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:0.0:1787829926.758931:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787829926.758934:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 02000000:00000001:2.0:1787829926.758935:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829926.758937:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787829926.758941:0:40701:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4727 02000000:00000001:2.0:1787829926.758941:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000100:00080000:0.0:1787829926.758946:0:40701:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787829926.758947:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829926.758948:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 02000000:00000001:2.0:1787829926.758949:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787829926.758950:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787829926.758953:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787829926.758955:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000020:00000001:0.0:1787829926.758956:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:0.0:1787829926.758957:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829926.758958:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:0.0:1787829926.758960:0:40701:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787829926.758962:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829926.758963:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787829926.758964:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787829926.758966:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829926.758967:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787829926.758968:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1787829926.758971:0:40701:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00000001:2.0:1787829926.758971:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601346880 : -126234108204736 : ffff8d30d4d26940) 00000008:00000001:0.0:1787829926.758975:0:40701:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829926.758978:0:40701:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829926.758979:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 02000000:00000001:0.0:1787829926.758981:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787829926.758984:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829926.758985:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787829926.758986:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787829926.758987:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829926.758991:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787829926.758995:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d25680 x1874670646878848/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787829926.758999:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787829926.759000:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829926.759003:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787829926.759004:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829926.759006:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1787829926.759008:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787829926.759008:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787829926.759010:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787829926.759011:0:40701:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4727 00000100:00080000:0.0:1787829926.759013:0:40701:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787829926.759015:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829926.759016:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787829926.759018:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787829926.759020:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787829926.759022:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000800:00000100:2.0:1787829926.759022:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000020:00000001:0.0:1787829926.759026:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:0.0:1787829926.759027:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829926.759028:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:0.0:1787829926.759031:0:40701:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787829926.759032:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829926.759033:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787829926.759035:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787829926.759036:0:2390:0:(events.c:34:request_out_callback()) Process entered 02000000:00000001:0.0:1787829926.759037:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829926.759040:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:0.0:1787829926.759041:0:40701:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00000001:2.0:1787829926.759043:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000008:00000001:0.0:1787829926.759044:0:40701:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829926.759047:0:40701:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829926.759049:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829926.759050:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787829926.759051:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829926.759052:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787829926.759053:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787829926.759054:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829926.759056:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787829926.759058:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d252c0 x1874670646878976/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787829926.759061:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787829926.759066:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787829926.759068:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829926.759069:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829926.759070:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829926.759078:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4299400960 00000100:00000001:2.0:1787829926.759087:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 00000100:00000001:2.0:1787829926.759094:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4299400960 00000100:00000001:2.0:1787829926.759098:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 00000100:00000001:2.0:1787829926.759101:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787829926.759104:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829926.759106:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1787829926.759113:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829926/real 1787829926] req@ffff8d30d4d26940 x1874670646878592/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829951 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1787829926.759123:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787829926.759125:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1787829926.759128:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1787829926.759130:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787829926.759132:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829926.759134:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787829926.759136:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787829926.759140:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d26940 x1874670646878592/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829951 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787829926.759147:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829926.759149:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787829926.759152:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829926.759154:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787829926.759155:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829926.759157:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787829926.759159:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787829926.759162:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829926.759166:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:2.0:1787829926.759168:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1787829926.759172:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d26940 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646878592:192.168.203.143@tcp:250:kworker.0 00000100:00000001:2.0:1787829926.759176:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829926.759178:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787829926.759180:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787829926.759182:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1787829926.759185:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787829926.759187:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:2.0:1787829926.759189:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:2.0:1787829926.759193:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787829926.759195:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1787829926.759198:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787829926.759199:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829926.759202:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829926.759208:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1787829926.759211:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829926.759212:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787829926.759214:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1787829926.759215:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1787829926.759217:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787829926.759218:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829926.759220:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787829926.759222:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787829926.759224:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787829926.759227:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d27c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646878720:192.168.203.143@tcp:38:kworker.0 00000100:00000001:2.0:1787829926.759231:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787829926.759233:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1787829926.759234:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829926.759235:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1787829926.759238:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829926.759243:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787829926.759244:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601351680 : -126234108199936 : ffff8d30d4d27c00) 00000100:00000001:2.0:1787829926.759248:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1787829926.759259:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:2.0:1787829926.759263:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1787829926.759265:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787829926.759266:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829926.759267:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1787829926.759270:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829926.759274:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4299399702 00000100:00000001:2.0:1787829926.759276:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4299400960 00000100:00000001:2.0:1787829926.759278:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4299399702 00000100:00000001:2.0:1787829926.759280:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4299400960 00000100:00000001:2.0:1787829926.759282:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787829926.759284:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829926.759286:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1787829926.759290:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829926/real 1787829926] req@ffff8d30d4d27c00 x1874670646878720/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829951 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1787829926.759297:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787829926.759298:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1787829926.759300:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1787829926.759302:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787829926.759303:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829926.759304:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787829926.759305:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787829926.759308:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d27c00 x1874670646878720/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829951 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787829926.759314:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829926.759316:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787829926.759318:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829926.759319:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787829926.759320:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829926.759322:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787829926.759323:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787829926.759325:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829926.759328:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:2.0:1787829926.759330:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1787829926.759332:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d27c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646878720:192.168.203.143@tcp:38:kworker.0 00000100:00000001:2.0:1787829926.759335:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1787829926.759336:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787829926.759337:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829926.759339:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787829926.759340:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787829926.759342:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787829926.759344:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d25680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646878848:192.168.203.143@tcp:38:kworker.0 00000100:00000001:2.0:1787829926.759349:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787829926.759350:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1787829926.759352:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829926.759353:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1787829926.759355:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829926.759359:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787829926.759360:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601342080 : -126234108209536 : ffff8d30d4d25680) 00000100:00000001:2.0:1787829926.759363:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1787829926.759371:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:2.0:1787829926.759374:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1787829926.759376:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787829926.759377:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829926.759379:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1787829926.759380:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829926.759383:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4299399702 00000100:00000001:2.0:1787829926.759385:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4299400960 00000100:00000001:2.0:1787829926.759387:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4299399702 00000100:00000001:2.0:1787829926.759389:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4299400960 00000100:00000001:2.0:1787829926.759390:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787829926.759392:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829926.759393:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1787829926.759396:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829926/real 1787829926] req@ffff8d30d4d25680 x1874670646878848/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829951 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1787829926.759403:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787829926.759404:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1787829926.759406:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1787829926.759407:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787829926.759408:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829926.759410:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787829926.759411:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787829926.759414:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d25680 x1874670646878848/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829951 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787829926.759426:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829926.759428:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787829926.759430:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829926.759431:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787829926.759433:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829926.759434:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787829926.759435:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787829926.759437:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829926.759440:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:2.0:1787829926.759442:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1787829926.759444:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d25680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646878848:192.168.203.143@tcp:38:kworker.0 00000100:00000001:2.0:1787829926.759455:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1787829926.759456:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787829926.759457:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829926.759460:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787829926.759461:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787829926.759463:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787829926.759465:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d252c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646878976:192.168.203.143@tcp:8:kworker.0 00000100:00000001:2.0:1787829926.759468:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787829926.759470:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1787829926.759471:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829926.759473:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1787829926.759475:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829926.759480:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787829926.759481:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601341120 : -126234108210496 : ffff8d30d4d252c0) 00000100:00000001:2.0:1787829926.759484:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1787829926.759494:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:2.0:1787829926.759498:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1787829926.759500:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787829926.759502:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829926.759503:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1787829926.759505:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829926.759508:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4299399715 00000100:00000001:2.0:1787829926.759511:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4299400960 00000100:00000001:2.0:1787829926.759513:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4299399715 00000100:00000001:2.0:1787829926.759515:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4299400960 00000100:00000001:2.0:1787829926.759517:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787829926.759518:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829926.759520:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1787829926.759524:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829926/real 1787829926] req@ffff8d30d4d252c0 x1874670646878976/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829951 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1787829926.759531:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787829926.759532:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1787829926.759533:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1787829926.759534:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787829926.759535:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829926.759536:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787829926.759537:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787829926.759539:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d252c0 x1874670646878976/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829951 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787829926.759543:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829926.759544:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787829926.759546:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829926.759546:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787829926.759547:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829926.759548:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787829926.759549:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787829926.759550:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829926.759552:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:2.0:1787829926.759553:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1787829926.759555:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d252c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646878976:192.168.203.143@tcp:8:kworker.0 00000100:00000001:2.0:1787829926.759557:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829926.759558:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787829926.759559:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787829926.759560:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1787829926.759561:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787829926.759562:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:2.0:1787829926.759563:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:2.0:1787829926.759564:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787829926.759565:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1787829926.759567:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787829926.759568:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829926.759569:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787829926.759570:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787829926.759570:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1787829926.759572:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787829926.759572:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:2.0:1787829926.759573:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:2.0:1787829926.759574:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787829926.759574:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1787829926.759576:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787829926.759577:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829926.759577:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787829926.759578:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787829926.759579:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1787829926.759579:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787829926.759580:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:2.0:1787829926.759580:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:2.0:1787829926.759581:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787829926.759582:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1787829926.759582:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787829926.759583:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829926.759584:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829926.759586:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1787829926.759587:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829926.759588:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787829926.759589:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829927.654708:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829927.654712:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829927.654772:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829927.654776:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829927.654779:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829928.678714:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829928.678720:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829928.678757:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829928.678760:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829928.678762:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829929.702712:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829929.702718:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829929.702784:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829929.702788:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829929.702791:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829930.726707:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829930.726712:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829930.726770:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829930.726773:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829930.726786:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829931.750686:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829931.750694:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829931.750749:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829931.750752:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829931.750755:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1787829931.878725:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829931.878736:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:0.0:1787829931.878738:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787829931.878742:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787829931.878747:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787829931.878754:0:40701:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 4732 00000100:00080000:0.0:1787829931.878764:0:40701:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787829931.878766:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829931.878767:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787829931.878770:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787829931.878772:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787829931.878774:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:0.0:1787829931.878776:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:0.0:1787829931.878777:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829931.878778:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:0.0:1787829931.878781:0:40701:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787829931.878782:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829931.878784:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787829931.878787:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787829931.878789:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829931.878790:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1787829931.878792:0:40701:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829931.878802:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787829931.878803:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829931.878806:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787829931.878809:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787829931.878812:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829931.878823:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787829931.878831:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d25a40 x1874670646879104/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787829931.878841:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787829931.878845:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829931.878864:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787829931.878873:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829931.878877:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:0.0:1787829931.878879:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787829931.878881:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787829931.878884:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787829931.878888:0:40701:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4732 00000100:00080000:0.0:1787829931.878893:0:40701:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787829931.878893:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787829931.878898:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829931.878900:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:2.0:1787829931.878900:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1787829931.878904:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000020:00000001:0.0:1787829931.878905:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787829931.878908:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:2.0:1787829931.878908:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787829931.878911:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:0.0:1787829931.878913:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000020:00000001:0.0:1787829931.878917:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 02000000:00000001:2.0:1787829931.878918:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000100:00000001:0.0:1787829931.878922:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 02000000:00000001:2.0:1787829931.878923:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787829931.878924:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 02000000:00000001:2.0:1787829931.878927:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787829931.878932:0:40701:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00100000:2.0:1787829931.878934:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d25a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646879104:192.168.203.143@tcp:250:kworker.0 00000100:00000001:0.0:1787829931.878936:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829931.878942:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:2.0:1787829931.878942:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787829931.878949:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787829931.878949:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787829931.878954:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829931.878954:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829931.878959:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 02000000:00000001:2.0:1787829931.878959:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 00000008:00100000:0.0:1787829931.878962:0:40701:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:0.0:1787829931.878965:0:40701:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829931.878966:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829931.878967:0:40701:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829931.878973:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787829931.878974:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829931.878975:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787829931.878977:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787829931.878978:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829931.878983:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787829931.878984:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829931.878987:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601343040 : -126234108208576 : ffff8d30d4d25a40) 00000100:00100000:0.0:1787829931.878991:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d270c0 x1874670646879232/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787829931.878992:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:0.0:1787829931.878999:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787829931.879001:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829931.879003:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787829931.879006:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829931.879008:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:0.0:1787829931.879009:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787829931.879010:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787829931.879012:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:0.0:1787829931.879020:0:40701:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4732 00000100:00080000:0.0:1787829931.879023:0:40701:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787829931.879025:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829931.879026:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:0.0:1787829931.879030:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000800:00000100:2.0:1787829931.879030:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000020:00000001:0.0:1787829931.879031:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:0.0:1787829931.879034:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000020:00000001:0.0:1787829931.879036:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:0.0:1787829931.879037:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829931.879038:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:0.0:1787829931.879041:0:40701:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787829931.879043:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829931.879044:0:2390:0:(events.c:34:request_out_callback()) Process entered 02000000:00000001:0.0:1787829931.879045:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:0.0:1787829931.879049:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787829931.879050:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 02000000:00000001:0.0:1787829931.879051:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829931.879054:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:2.0:1787829931.879055:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000008:00100000:0.0:1787829931.879056:0:40701:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00000001:2.0:1787829931.879059:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000008:00000001:0.0:1787829931.879060:0:40701:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829931.879063:0:40701:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829931.879066:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829931.879067:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787829931.879071:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829931.879072:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4299406080 00000100:00000001:0.0:1787829931.879073:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787829931.879077:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787829931.879079:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829931.879084:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1787829931.879085:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829931.879092:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4299406080 00000100:00100000:0.0:1787829931.879093:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d24000 x1874670646879360/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787829931.879100:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 00000100:00000001:2.0:1787829931.879104:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787829931.879109:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829931.879114:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00000001:0.0:1787829931.879118:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787829931.879121:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787829931.879122:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829931/real 1787829931] req@ffff8d30d4d25a40 x1874670646879104/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829956 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787829931.879124:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787829931.879128:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1787829931.879135:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1787829931.879137:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:0.0:1787829931.879139:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787829931.879142:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000400:00000001:2.0:1787829931.879143:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000100:00080000:0.0:1787829931.879146:0:40701:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4732 00000400:00000001:2.0:1787829931.879146:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00080000:0.0:1787829931.879152:0:40701:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:2.0:1787829931.879152:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787829931.879156:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829931.879157:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:2.0:1787829931.879157:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787829931.879160:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000020:00000001:0.0:1787829931.879161:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:0.0:1787829931.879164:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:2.0:1787829931.879165:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000020:00000001:0.0:1787829931.879166:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000020:00000001:0.0:1787829931.879169:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:2.0:1787829931.879169:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829931.879174:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829931.879176:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00100000:2.0:1787829931.879178:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d25a40 x1874670646879104/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829956 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1787829931.879180:0:40701:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787829931.879184:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829931.879188:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:2.0:1787829931.879192:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829931.879195:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 02000000:00000001:0.0:1787829931.879198:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787829931.879200:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:0.0:1787829931.879203:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829931.879204:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787829931.879205:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:2.0:1787829931.879207:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000008:00100000:0.0:1787829931.879208:0:40701:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000008:00000001:0.0:1787829931.879211:0:40701:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829931.879211:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787829931.879215:0:40701:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829931.879215:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 02000000:00000001:0.0:1787829931.879221:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00080000:2.0:1787829931.879221:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 02000000:00000001:0.0:1787829931.879223:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829931.879226:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787829931.879229:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00080000:2.0:1787829931.879229:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 02000000:00000001:0.0:1787829931.879232:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829931.879235:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00000001:0.0:1787829931.879237:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787829931.879241:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d25a40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646879104:192.168.203.143@tcp:250:kworker.0 00000100:00100000:0.0:1787829931.879245:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30d4d24f00 x1874670646879488/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:2.0:1787829931.879248:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829931.879253:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787829931.879254:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:2.0:1787829931.879256:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 00000100:00000001:0.0:1787829931.879257:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829931.879261:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 00000100:00000001:0.0:1787829931.879262:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829931.879267:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787829931.879270:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:2.0:1787829931.879272:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:2.0:1787829931.879276:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787829931.879277:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1787829931.879281:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787829931.879283:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829931.879286:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829931.879292:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1787829931.879294:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829931.879296:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787829931.879298:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:2.0:1787829931.879300:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1787829931.879301:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787829931.879302:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829931.879304:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787829931.879305:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787829931.879308:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787829931.879311:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d270c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646879232:192.168.203.143@tcp:38:kworker.0 00000100:00000001:2.0:1787829931.879314:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787829931.879316:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1787829931.879318:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829931.879319:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1787829931.879321:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829931.879326:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787829931.879328:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601348800 : -126234108202816 : ffff8d30d4d270c0) 00000100:00000001:2.0:1787829931.879332:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1787829931.879342:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:2.0:1787829931.879346:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1787829931.879348:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787829931.879349:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829931.879351:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1787829931.879353:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829931.879357:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4299404822 00000100:00000001:2.0:1787829931.879359:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4299406080 00000100:00000001:2.0:1787829931.879362:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4299404822 00000100:00000001:2.0:1787829931.879364:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4299406080 00000100:00000001:2.0:1787829931.879366:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787829931.879368:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829931.879370:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1787829931.879373:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829931/real 1787829931] req@ffff8d30d4d270c0 x1874670646879232/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829956 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1787829931.879382:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787829931.879383:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1787829931.879385:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1787829931.879387:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787829931.879388:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829931.879389:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787829931.879391:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787829931.879394:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d270c0 x1874670646879232/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829956 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787829931.879402:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829931.879404:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787829931.879406:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829931.879408:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787829931.879409:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829931.879410:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787829931.879412:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787829931.879414:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829931.879416:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:2.0:1787829931.879418:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1787829931.879421:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d270c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646879232:192.168.203.143@tcp:38:kworker.0 00000100:00000001:2.0:1787829931.879425:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1787829931.879426:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787829931.879427:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829931.879429:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787829931.879431:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787829931.879432:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787829931.879435:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d24000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646879360:192.168.203.143@tcp:38:kworker.0 00000100:00000001:2.0:1787829931.879438:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787829931.879439:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1787829931.879441:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829931.879443:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1787829931.879445:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829931.879449:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787829931.879450:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601336320 : -126234108215296 : ffff8d30d4d24000) 00000100:00000001:2.0:1787829931.879453:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1787829931.879462:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:2.0:1787829931.879465:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1787829931.879467:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787829931.879468:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829931.879470:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1787829931.879472:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829931.879474:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4299404822 00000100:00000001:2.0:1787829931.879477:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4299406080 00000100:00000001:2.0:1787829931.879479:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4299404822 00000100:00000001:2.0:1787829931.879481:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4299406080 00000100:00000001:2.0:1787829931.879483:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787829931.879485:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829931.879486:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1787829931.879490:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829931/real 1787829931] req@ffff8d30d4d24000 x1874670646879360/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829956 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1787829931.879497:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787829931.879498:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1787829931.879500:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1787829931.879502:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787829931.879503:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829931.879504:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787829931.879506:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787829931.879509:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d24000 x1874670646879360/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829956 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787829931.879516:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829931.879517:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787829931.879520:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829931.879521:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787829931.879522:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829931.879524:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787829931.879525:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787829931.879527:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829931.879530:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:2.0:1787829931.879533:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1787829931.879536:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d24000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646879360:192.168.203.143@tcp:38:kworker.0 00000100:00000001:2.0:1787829931.879539:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:2.0:1787829931.879540:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:2.0:1787829931.879541:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829931.879543:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:2.0:1787829931.879544:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:2.0:1787829931.879546:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:2.0:1787829931.879548:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30d4d24f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646879488:192.168.203.143@tcp:8:kworker.0 00000100:00000001:2.0:1787829931.879552:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:2.0:1787829931.879553:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:2.0:1787829931.879555:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:2.0:1787829931.879556:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:2.0:1787829931.879558:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829931.879563:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:2.0:1787829931.879564:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839601340160 : -126234108211456 : ffff8d30d4d24f00) 00000100:00000001:2.0:1787829931.879567:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:2.0:1787829931.879575:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:2.0:1787829931.879578:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:2.0:1787829931.879579:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787829931.879581:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829931.879582:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:2.0:1787829931.879584:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829931.879587:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4299404835 00000100:00000001:2.0:1787829931.879589:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4299406080 00000100:00000001:2.0:1787829931.879592:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4299404835 00000100:00000001:2.0:1787829931.879594:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4299406080 00000100:00000001:2.0:1787829931.879596:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:2.0:1787829931.879598:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829931.879599:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:2.0:1787829931.879636:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829931/real 1787829931] req@ffff8d30d4d24f00 x1874670646879488/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829956 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:2.0:1787829931.879644:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:2.0:1787829931.879645:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:2.0:1787829931.879649:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:2.0:1787829931.879650:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:2.0:1787829931.879652:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829931.879654:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787829931.879655:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:2.0:1787829931.879659:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30d4d24f00 x1874670646879488/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829956 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:2.0:1787829931.879665:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829931.879667:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:2.0:1787829931.879669:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829931.879671:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:2.0:1787829931.879672:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829931.879674:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:2.0:1787829931.879676:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:2.0:1787829931.879678:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1787829931.879680:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:2.0:1787829931.879682:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:2.0:1787829931.879685:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30d4d24f00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646879488:192.168.203.143@tcp:8:kworker.0 00000100:00000001:2.0:1787829931.879689:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829931.879691:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787829931.879692:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787829931.879694:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1787829931.879696:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787829931.879697:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:2.0:1787829931.879698:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:2.0:1787829931.879700:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787829931.879701:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1787829931.879704:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787829931.879705:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829931.879714:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787829931.879715:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787829931.879716:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1787829931.879718:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787829931.879719:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:2.0:1787829931.879720:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:2.0:1787829931.879721:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787829931.879722:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1787829931.879724:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787829931.879726:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829931.879727:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:2.0:1787829931.879728:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:2.0:1787829931.879729:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:2.0:1787829931.879731:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:2.0:1787829931.879732:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:2.0:1787829931.879733:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:2.0:1787829931.879734:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:2.0:1787829931.879735:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:2.0:1787829931.879737:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:2.0:1787829931.879737:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829931.879739:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:2.0:1787829931.879742:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:2.0:1787829931.879743:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:2.0:1787829931.879745:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:2.0:1787829931.879750:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829932.774676:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829932.774680:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829932.774736:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829932.774740:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829932.774744:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829933.798702:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829933.798709:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829933.798789:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829933.798792:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829933.798795:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829934.822739:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829934.822744:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829934.822815:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829934.822818:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829934.822821:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829935.846710:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829935.846716:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829935.846856:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829935.846860:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829935.846863:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829936.870679:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829936.870686:0:815:0:(ldlm_request.c:2324:ldlm_cancel_lru()) Process entered 00010000:00000001:2.0:1787829936.870688:0:815:0:(ldlm_request.c:2112:ldlm_prepare_lru_list()) Process entered 00010000:00000001:2.0:1787829936.870691:0:815:0:(ldlm_request.c:2284:ldlm_prepare_lru_list()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829936.870696:0:815:0:(ldlm_lockd.c:2241:ldlm_bl_to_thread()) Process entered 00010000:00000001:2.0:1787829936.870698:0:815:0:(ldlm_lockd.c:2244:ldlm_bl_to_thread()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829936.870700:0:815:0:(ldlm_request.c:2334:ldlm_cancel_lru()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829936.870703:0:815:0:(ldlm_pool.c:496:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829936.870744:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829936.870748:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829936.870751:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787829936.998687:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829936.998697:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1787829936.998700:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787829936.998703:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787829936.998707:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787829936.998714:0:40701:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 4737 00000100:00080000:3.0:1787829936.998724:0:40701:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787829936.998726:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829936.998728:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:3.0:1787829936.998733:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1787829936.998735:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1787829936.998739:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:3.0:1787829936.998742:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:3.0:1787829936.998744:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829936.998745:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:3.0:1787829936.998750:0:40701:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:3.0:1787829936.998752:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829936.998755:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787829936.998758:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829936.998760:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829936.998763:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:3.0:1787829936.998765:0:40701:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829936.998772:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787829936.998774:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829936.998777:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787829936.998780:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787829936.998782:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829936.998790:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829936.998804:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c7014b40 x1874670646879616/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787829936.998814:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787829936.998818:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829936.998838:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787829936.998842:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829936.998846:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1787829936.998848:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787829936.998859:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787829936.998862:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:0.0:1787829936.998863:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00080000:3.0:1787829936.998866:0:40701:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4737 00000100:00000001:0.0:1787829936.998869:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00080000:3.0:1787829936.998871:0:40701:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787829936.998873:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1787829936.998877:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787829936.998880:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829936.998885:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787829936.998888:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787829936.998891:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787829936.998898:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c7014b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646879616:192.168.203.143@tcp:250:kworker.0 00000100:00000001:0.0:1787829936.998903:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787829936.998907:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787829936.998910:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829936.998912:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1787829936.998917:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829936.998925:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829936.998927:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787829936.998929:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:0.0:1787829936.998930:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369538368 : -126234340013248 : ffff8d30c7014b40) 00000020:00000001:3.0:1787829936.998935:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:0.0:1787829936.998936:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000020:00000001:3.0:1787829936.998938:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000800:00000100:0.0:1787829936.998964:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:0.0:1787829936.998971:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1787829936.998974:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787829936.998975:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829936.998976:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787829936.998980:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829936.998983:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4299411200 00000100:00000001:0.0:1787829936.998989:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 00000100:00000001:0.0:1787829936.998994:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4299411200 00000100:00000001:0.0:1787829936.998997:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 00000100:00000001:0.0:1787829936.998999:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787829936.999000:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829936.999002:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787829936.999006:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829936/real 1787829936] req@ffff8d30c7014b40 x1874670646879616/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829961 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000020:00000001:3.0:1787829936.999021:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000400:00000001:0.0:1787829936.999022:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000020:00000001:3.0:1787829936.999025:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000400:00000001:0.0:1787829936.999027:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787829936.999032:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787829936.999034:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787829936.999036:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829936.999038:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787829936.999040:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787829936.999045:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c7014b40 x1874670646879616/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829961 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787829936.999052:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829936.999055:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787829936.999058:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829936.999059:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787829936.999061:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829936.999063:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787829936.999064:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787829936.999067:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829936.999071:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787829936.999073:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787829936.999077:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c7014b40 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646879616:192.168.203.143@tcp:250:kworker.0 00000100:00000001:0.0:1787829936.999081:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829936.999083:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787829936.999084:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787829936.999086:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787829936.999088:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787829936.999090:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787829936.999092:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787829936.999096:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787829936.999097:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787829936.999099:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787829936.999101:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829936.999103:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829936.999109:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787829936.999111:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829936.999113:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787829936.999115:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829936.999226:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829936.999227:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:3.0:1787829936.999231:0:40701:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:3.0:1787829936.999233:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829936.999234:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787829936.999239:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829936.999240:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829936.999242:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:3.0:1787829936.999254:0:40701:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1787829936.999256:0:40701:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829936.999258:0:40701:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829936.999261:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787829936.999261:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829936.999263:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787829936.999264:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787829936.999266:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829936.999276:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829936.999281:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c70161c0 x1874670646879744/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787829936.999335:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787829936.999337:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829936.999343:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787829936.999345:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829936.999348:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1787829936.999348:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787829936.999350:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787829936.999351:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787829936.999353:0:40701:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4737 00000100:00080000:3.0:1787829936.999366:0:40701:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787829936.999368:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829936.999369:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:0.0:1787829936.999370:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000020:00000001:3.0:1787829936.999371:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1787829936.999374:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:0.0:1787829936.999375:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000020:00000001:3.0:1787829936.999376:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:0.0:1787829936.999379:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000020:00000001:3.0:1787829936.999379:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:3.0:1787829936.999383:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829936.999384:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:3.0:1787829936.999384:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:0.0:1787829936.999387:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787829936.999389:0:40701:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 02000000:00000001:0.0:1787829936.999394:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787829936.999398:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787829936.999400:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829936.999404:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787829936.999405:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c70161c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646879744:192.168.203.143@tcp:38:kworker.0 02000000:00000001:3.0:1787829936.999407:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787829936.999410:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787829936.999411:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1787829936.999414:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829936.999416:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:3.0:1787829936.999417:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 02000000:00000001:0.0:1787829936.999421:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 00000008:00100000:3.0:1787829936.999421:0:40701:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 02000000:00000001:0.0:1787829936.999425:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1787829936.999428:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000008:00000001:3.0:1787829936.999433:0:40701:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829936.999434:0:40701:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829936.999438:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 02000000:00000001:3.0:1787829936.999438:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787829936.999442:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829936.999443:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369544128 : -126234340007488 : ffff8d30c70161c0) 00000100:00000001:3.0:1787829936.999443:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787829936.999445:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787829936.999447:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829936.999451:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:3.0:1787829936.999452:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829936.999463:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c70152c0 x1874670646879872/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1787829936.999471:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:3.0:1787829936.999471:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787829936.999474:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829936.999478:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829936.999480:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00080000:3.0:1787829936.999480:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1787829936.999485:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787829936.999486:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829936.999487:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787829936.999491:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787829936.999492:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:3.0:1787829936.999495:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787829936.999497:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:0.0:1787829936.999498:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4299409942 00000100:00000001:3.0:1787829936.999499:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00000001:0.0:1787829936.999501:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4299411200 00000100:00080000:3.0:1787829936.999503:0:40701:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4737 00000100:00000001:0.0:1787829936.999505:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4299409942 00000100:00080000:3.0:1787829936.999507:0:40701:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787829936.999508:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4299411200 00000100:00000001:3.0:1787829936.999511:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829936.999512:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787829936.999514:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829936.999515:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787829936.999520:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829936/real 1787829936] req@ffff8d30c70161c0 x1874670646879744/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829961 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829936.999520:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:3.0:1787829936.999524:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1787829936.999526:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1787829936.999529:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000400:00000001:0.0:1787829936.999533:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000020:00000001:3.0:1787829936.999533:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00000001:3.0:1787829936.999535:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000400:00000001:0.0:1787829936.999536:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:3.0:1787829936.999537:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:0.0:1787829936.999539:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787829936.999541:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787829936.999542:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829936.999543:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787829936.999544:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787829936.999548:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c70161c0 x1874670646879744/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829961 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1787829936.999550:0:40701:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:3.0:1787829936.999554:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829936.999558:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787829936.999560:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829936.999565:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829936.999566:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829936.999567:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000008:00100000:3.0:1787829936.999569:0:40701:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00000001:0.0:1787829936.999571:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787829936.999574:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829936.999575:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000008:00000001:3.0:1787829936.999578:0:40701:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829936.999579:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829936.999580:0:40701:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829936.999594:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787829936.999597:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787829936.999618:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829936.999623:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787829936.999626:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 02000000:00000001:3.0:1787829936.999628:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000100:00100000:0.0:1787829936.999630:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c70161c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646879744:192.168.203.143@tcp:38:kworker.0 02000000:00000001:3.0:1787829936.999631:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829936.999634:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787829936.999636:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 00000100:00000001:0.0:1787829936.999639:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 02000000:00000001:3.0:1787829936.999640:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829936.999644:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787829936.999648:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 00000100:00000001:3.0:1787829936.999649:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829936.999650:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787829936.999655:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00100000:3.0:1787829936.999657:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c70143c0 x1874670646880000/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000020:00000001:0.0:1787829936.999658:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787829936.999661:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787829936.999665:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787829936.999667:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:3.0:1787829936.999670:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:0.0:1787829936.999671:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787829936.999675:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:3.0:1787829936.999675:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829936.999677:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829936.999680:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787829936.999682:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829936.999684:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787829936.999685:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787829936.999686:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1787829936.999688:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787829936.999691:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829936.999692:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829936.999693:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787829936.999697:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787829936.999699:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787829936.999702:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c70152c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646879872:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787829936.999704:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787829936.999706:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787829936.999708:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829936.999709:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1787829936.999711:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829936.999715:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787829936.999716:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369540288 : -126234340011328 : ffff8d30c70152c0) 00000100:00000001:0.0:1787829936.999719:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1787829936.999728:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:0.0:1787829936.999731:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1787829936.999733:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787829936.999734:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829936.999735:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787829936.999736:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829936.999739:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4299409943 00000100:00000001:0.0:1787829936.999741:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4299411200 00000100:00000001:0.0:1787829936.999743:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4299409943 00000100:00000001:0.0:1787829936.999745:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4299411200 00000100:00000001:0.0:1787829936.999747:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787829936.999748:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829936.999750:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787829936.999754:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829936/real 1787829936] req@ffff8d30c70152c0 x1874670646879872/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829961 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787829936.999762:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787829936.999763:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787829936.999765:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787829936.999767:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787829936.999768:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829936.999770:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787829936.999771:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787829936.999774:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c70152c0 x1874670646879872/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829961 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787829936.999781:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829936.999784:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787829936.999786:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829936.999788:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787829936.999789:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829936.999790:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787829936.999792:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787829936.999794:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829936.999797:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787829936.999798:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787829936.999800:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c70152c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646879872:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787829936.999803:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829936.999804:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787829936.999804:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787829936.999805:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787829936.999807:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787829936.999808:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787829936.999809:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787829936.999810:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787829936.999811:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787829936.999812:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787829936.999813:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829936.999814:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829936.999815:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787829936.999816:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829936.999817:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787829936.999818:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787829936.999819:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1787829936.999820:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787829936.999821:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829936.999822:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787829936.999824:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787829936.999825:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787829936.999827:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c70143c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646880000:192.168.203.143@tcp:8:kworker.0 00000100:00000001:0.0:1787829936.999829:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787829936.999830:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787829936.999831:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829936.999832:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1787829936.999833:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829936.999837:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787829936.999838:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369536448 : -126234340015168 : ffff8d30c70143c0) 00000100:00000001:0.0:1787829936.999841:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1787829936.999848:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:0.0:1787829936.999851:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1787829936.999852:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787829936.999853:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829936.999854:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787829936.999855:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829936.999857:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4299409956 00000100:00000001:0.0:1787829936.999859:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4299411200 00000100:00000001:0.0:1787829936.999860:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4299409956 00000100:00000001:0.0:1787829936.999861:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4299411200 00000100:00000001:0.0:1787829936.999862:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787829936.999864:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829936.999865:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787829936.999867:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829936/real 1787829936] req@ffff8d30c70143c0 x1874670646880000/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829961 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787829936.999872:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787829936.999872:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787829936.999874:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787829936.999875:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787829936.999875:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829936.999876:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787829936.999877:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787829936.999879:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c70143c0 x1874670646880000/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829961 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787829936.999883:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829936.999884:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787829936.999885:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829936.999886:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787829936.999887:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829936.999888:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787829936.999889:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787829936.999890:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829936.999892:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787829936.999893:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787829936.999894:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c70143c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646880000:192.168.203.143@tcp:8:kworker.0 00000100:00000001:0.0:1787829936.999896:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829936.999897:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787829936.999897:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787829936.999898:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787829936.999899:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787829936.999900:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787829936.999900:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787829936.999901:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787829936.999902:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787829936.999903:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787829936.999904:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829936.999905:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829936.999907:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787829936.999908:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829936.999909:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787829936.999909:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829937.894704:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829937.894711:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829937.894760:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829937.894764:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829937.894766:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829938.918662:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829938.918668:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829938.918720:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829938.918723:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829938.918726:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829939.942686:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829939.942692:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829939.942722:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829939.942725:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829939.942728:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829940.966660:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829940.966665:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829940.966720:0:6161:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829940.966724:0:6161:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829940.966727:0:6161:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00000001:2.0:1787829941.990717:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829941.990724:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:1.0:1787829941.990774:0:6162:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:1.0:1787829941.990777:0:6162:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:1.0:1787829941.990780:0:6162:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1787829942.118697:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7437a6eb-050e-432b-aaf0-5fa0c62800f9->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829942.118705:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00000001:3.0:1787829942.118707:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787829942.118710:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787829942.118714:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787829942.118720:0:40701:0:(import.c:510:import_select_connection()) MGC192.168.203.143@tcp: connect to NID 192.168.203.143@tcp last attempt 4742 00000100:00080000:3.0:1787829942.118729:0:40701:0:(import.c:589:import_select_connection()) MGC192.168.203.143@tcp: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787829942.118731:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829942.118732:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:3.0:1787829942.118735:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1787829942.118738:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1787829942.118740:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000020:00000001:3.0:1787829942.118742:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839534592000 : -126234174959616 : ffff8d30d0d7d000) 00000100:00000001:3.0:1787829942.118743:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829942.118744:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00080000:3.0:1787829942.118747:0:40701:0:(import.c:624:import_select_connection()) MGC192.168.203.143@tcp: import ffff8d30d0081000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:3.0:1787829942.118748:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829942.118750:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 02000000:00000001:3.0:1787829942.118752:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829942.118754:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829942.118756:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:3.0:1787829942.118757:0:40701:0:(obd_class.h:895:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829942.118765:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787829942.118766:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829942.118768:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787829942.118770:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787829942.118773:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829942.118780:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829942.118786:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c7016d00 x1874670646880128/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:3.0:1787829942.118793:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787829942.118796:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829942.118804:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787829942.118806:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829942.118808:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00000001:3.0:1787829942.118809:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787829942.118810:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787829942.118813:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000100:00080000:3.0:1787829942.118814:0:40701:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4742 00000100:00080000:3.0:1787829942.118817:0:40701:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:3.0:1787829942.118818:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829942.118819:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000020:00000001:3.0:1787829942.118821:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000020:00000001:3.0:1787829942.118822:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000020:00000001:3.0:1787829942.118824:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000020:00000001:3.0:1787829942.118825:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545694208 : -126234163857408 : ffff8d30d1813800) 00000100:00000001:3.0:1787829942.118826:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829942.118827:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:3.0:1787829942.118828:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:0.0:1787829942.118831:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00080000:3.0:1787829942.118832:0:40701:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d30c675d800: import ffff8d30d0084800 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:3.0:1787829942.118835:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829942.118836:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 02000000:00000001:3.0:1787829942.118837:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:0.0:1787829942.118838:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787829942.118840:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829942.118842:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787829942.118844:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:3.0:1787829942.118845:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829942.118847:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 00000100:00000001:3.0:1787829942.118848:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 02000000:00000001:0.0:1787829942.118851:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000008:00100000:3.0:1787829942.118853:0:40701:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1787829942.118856:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c7016d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646880128:192.168.203.143@tcp:250:kworker.0 00000008:00000001:3.0:1787829942.118857:0:40701:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829942.118861:0:40701:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829942.118863:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:3.0:1787829942.118865:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:0.0:1787829942.118867:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:3.0:1787829942.118867:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829942.118869:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:0.0:1787829942.118870:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829942.118871:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:0.0:1787829942.118873:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:3.0:1787829942.118874:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829942.118877:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829942.118881:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:3.0:1787829942.118888:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c70170c0 x1874670646880256/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787829942.118892:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787829942.118894:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369547008 : -126234340004608 : ffff8d30c7016d00) 00000100:00000001:0.0:1787829942.118896:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000100:00000001:3.0:1787829942.118905:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787829942.118907:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829942.118910:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787829942.118913:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1787829942.118917:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00000001:3.0:1787829942.118918:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 00000100:00080000:3.0:1787829942.118920:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00000001:3.0:1787829942.118923:0:40701:0:(import.c:485:import_select_connection()) Process entered 00000800:00000100:0.0:1787829942.118924:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00080000:3.0:1787829942.118928:0:40701:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4742 00000100:00000001:0.0:1787829942.118934:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00080000:3.0:1787829942.118934:0:40701:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787829942.118939:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:3.0:1787829942.118939:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829942.118942:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829942.118943:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:0.0:1787829942.118945:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000020:00000001:3.0:1787829942.118948:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:0.0:1787829942.118950:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829942.118951:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 00000100:00000001:0.0:1787829942.118954:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 0 is 4299416320 00000020:00000001:3.0:1787829942.118955:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000020:00000001:3.0:1787829942.118961:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839545706496 : -126234163845120 : ffff8d30d1816800) 00000100:00000001:0.0:1787829942.118964:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 0 00000100:00000001:3.0:1787829942.118964:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:3.0:1787829942.118967:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:0.0:1787829942.118970:0:2390:0:(niobuf.c:663:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS new deadline estimation for cpu 1 is 4299416320 00000100:00000001:0.0:1787829942.118974:0:2390:0:(niobuf.c:678:kick_cpu_latency()) MGC192.168.203.143@tcp: PM QoS request now active for cpu 1 00000100:00080000:3.0:1787829942.118974:0:40701:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d30c675d800: import ffff8d30c9f79000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:3.0:1787829942.118979:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829942.118980:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829942.118981:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:0.0:1787829942.118985:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:3.0:1787829942.118986:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787829942.118987:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 02000000:00000001:3.0:1787829942.118990:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829942.118993:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00100000:0.0:1787829942.118994:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829942/real 1787829942] req@ffff8d30c7016d00 x1874670646880128/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829967 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1787829942.118996:0:40701:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683205 ocd_grant: 2097152, lost: 0. 00000008:00000001:3.0:1787829942.119001:0:40701:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829942.119004:0:40701:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:0.0:1787829942.119010:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 02000000:00000001:3.0:1787829942.119010:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 00000400:00000001:0.0:1787829942.119012:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 02000000:00000001:3.0:1787829942.119013:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829942.119016:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:3.0:1787829942.119016:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787829942.119017:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787829942.119020:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829942.119021:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787829942.119023:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829942.119026:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:3.0:1787829942.119027:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829942.119028:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:3.0:1787829942.119037:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c7017840 x1874670646880384/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1787829942.119038:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c7016d00 x1874670646880128/t0(0) o250->MGC192.168.203.143@tcp@192.168.203.143@tcp:26/25 lens 520/544 e 0 to 1 dl 1787829967 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787829942.119058:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829942.119061:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787829942.119066:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829942.119069:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787829942.119070:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829942.119074:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787829942.119076:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:3.0:1787829942.119076:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00080000:0.0:1787829942.119080:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0081000 MGS: changing import state from CONNECTING to DISCONN 00000100:00000001:3.0:1787829942.119081:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:0.0:1787829942.119084:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.143@tcp failed (-110) 00000100:00000001:3.0:1787829942.119086:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829942.119087:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787829942.119093:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c7016d00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7437a6eb-050e-432b-aaf0-5fa0c62800f9:2390:1874670646880128:192.168.203.143@tcp:250:kworker.0 00000100:00080000:3.0:1787829942.119094:0:40701:0:(pinger.c:229:ptlrpc_pinger_process_import()) 304cf071-b03b-42d4-83cb-76e070d32538->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00000001:0.0:1787829942.119098:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829942.119100:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00080000:3.0:1787829942.119100:0:40701:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00000001:0.0:1787829942.119103:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 00000100:00000001:3.0:1787829942.119104:0:40701:0:(import.c:701:ptlrpc_connect_import_locked()) Process entered 02000000:00000001:0.0:1787829942.119105:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787829942.119109:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000100:00080000:3.0:1787829942.119109:0:40701:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000020:00000001:0.0:1787829942.119111:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787829942.119113:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 00000100:00000001:3.0:1787829942.119116:0:40701:0:(import.c:485:import_select_connection()) Process entered 02000000:00000001:0.0:1787829942.119118:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787829942.119120:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787829942.119123:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00080000:3.0:1787829942.119123:0:40701:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: connect to NID 192.168.203.143@tcp last attempt 4742 00000100:00000001:0.0:1787829942.119125:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829942.119128:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00080000:3.0:1787829942.119131:0:40701:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: tried all connections, increasing latency to 20s 00000100:00000001:0.0:1787829942.119134:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:3.0:1787829942.119135:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829942.119137:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829942.119139:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 00000100:00000001:0.0:1787829942.119140:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787829942.119141:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787829942.119143:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1787829942.119145:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000020:00000001:3.0:1787829942.119145:0:40701:0:(genops.c:792:class_conn2export()) Process entered 00000100:00000001:0.0:1787829942.119150:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829942.119150:0:40701:0:(lustre_handles.c:130:class_handle2object()) Process entered 02000000:00000001:0.0:1787829942.119154:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 00000020:00000001:3.0:1787829942.119154:0:40701:0:(lustre_handles.c:155:class_handle2object()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 02000000:00000001:0.0:1787829942.119155:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787829942.119160:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829942.119160:0:40701:0:(genops.c:806:class_conn2export()) Process leaving (rc=18446617839419256832 : -126234290294784 : ffff8d30c9f7f000) 00000100:00100000:0.0:1787829942.119166:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c70170c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646880256:192.168.203.143@tcp:38:kworker.0 00000100:00000001:3.0:1787829942.119166:0:40701:0:(connection.c:156:ptlrpc_connection_addref()) Process entered 00000100:00000001:0.0:1787829942.119169:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787829942.119171:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 00000100:00000001:3.0:1787829942.119171:0:40701:0:(connection.c:163:ptlrpc_connection_addref()) Process leaving (rc=18446617839521529984 : -126234188021632 : ffff8d30d0108080) 02000000:00000001:0.0:1787829942.119173:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829942.119176:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1787829942.119177:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00080000:3.0:1787829942.119179:0:40701:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d30c675d800: import ffff8d30c9f7d000 using connection 192.168.203.143@tcp/192.168.203.143@tcp 00000100:00000001:0.0:1787829942.119181:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:3.0:1787829942.119183:0:40701:0:(import.c:628:import_select_connection()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829942.119184:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369547968 : -126234340003648 : ffff8d30c70170c0) 02000000:00000001:3.0:1787829942.119189:0:40701:0:(sec.c:1579:sptlrpc_import_sec_adapt()) Process entered 00000100:00000001:0.0:1787829942.119190:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 02000000:00000001:3.0:1787829942.119193:0:40701:0:(sec.c:1619:sptlrpc_import_sec_adapt()) Process leaving via out (rc=0 : 0 : 0x0) 02000000:00000001:3.0:1787829942.119197:0:40701:0:(sec.c:1647:sptlrpc_import_sec_adapt()) Process leaving (rc=0 : 0 : 0) 00000800:00000100:0.0:1787829942.119201:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:3.0:1787829942.119201:0:40701:0:(obd_class.h:888:obd_reconnect()) Process entered 00000100:00000001:0.0:1787829942.119205:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000008:00100000:3.0:1787829942.119206:0:40701:0:(osc_request.c:4019:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683205 ocd_grant: 8388608, lost: 0. 00000100:00000001:0.0:1787829942.119207:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787829942.119209:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829942.119212:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000008:00000001:3.0:1787829942.119213:0:40701:0:(osc_request.c:4022:osc_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829942.119215:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829942.119216:0:40701:0:(obd_class.h:902:obd_reconnect()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829942.119219:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4299415062 02000000:00000001:3.0:1787829942.119221:0:40701:0:(sec.c:2038:sptlrpc_sepol_get()) Process entered 02000000:00000001:3.0:1787829942.119223:0:40701:0:(sec.c:2048:sptlrpc_sepol_get()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829942.119224:0:40701:0:(client.c:858:ptlrpc_request_bufs_pack()) Process entered 02000000:00000001:3.0:1787829942.119225:0:40701:0:(sec.c:433:sptlrpc_req_get_ctx()) Process entered 02000000:00000001:3.0:1787829942.119227:0:40701:0:(sec.c:457:sptlrpc_req_get_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829942.119231:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4299416320 00000100:00000001:3.0:1787829942.119233:0:40701:0:(client.c:938:ptlrpc_request_bufs_pack()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829942.119234:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4299415062 00000100:00000001:0.0:1787829942.119239:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0000-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4299416320 00000100:00000001:0.0:1787829942.119241:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00100000:3.0:1787829942.119242:0:40701:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d30c7014000 x1874670646880512/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00000001:0.0:1787829942.119243:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829942.119247:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787829942.119253:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829942/real 1787829942] req@ffff8d30c70170c0 x1874670646880256/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829967 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:3.0:1787829942.119254:0:40701:0:(jobid.c:939:lustre_get_jobid()) Process entered 00000100:00000001:3.0:1787829942.119257:0:40701:0:(jobid.c:979:lustre_get_jobid()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:3.0:1787829942.119262:0:40701:0:(import.c:833:ptlrpc_connect_import_locked()) Process leaving (rc=0 : 0 : 0) 00000400:00000001:0.0:1787829942.119266:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787829942.119268:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787829942.119271:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787829942.119272:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787829942.119273:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829942.119275:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787829942.119276:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787829942.119279:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c70170c0 x1874670646880256/t0(0) o38->lustre-MDT0000-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829967 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787829942.119284:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829942.119285:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787829942.119287:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829942.119288:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787829942.119289:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829942.119289:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787829942.119290:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787829942.119292:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30d0084800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829942.119294:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787829942.119295:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787829942.119297:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c70170c0 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646880256:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787829942.119299:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1787829942.119301:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787829942.119302:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829942.119303:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787829942.119305:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787829942.119306:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787829942.119308:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c7017840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646880384:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787829942.119311:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787829942.119312:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787829942.119313:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829942.119314:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1787829942.119316:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829942.119319:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787829942.119319:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369549888 : -126234340001728 : ffff8d30c7017840) 00000100:00000001:0.0:1787829942.119321:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1787829942.119327:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:0.0:1787829942.119329:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1787829942.119330:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787829942.119331:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829942.119332:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787829942.119333:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829942.119335:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4299415062 00000100:00000001:0.0:1787829942.119337:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4299416320 00000100:00000001:0.0:1787829942.119339:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4299415062 00000100:00000001:0.0:1787829942.119341:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-MDT0001-mdc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4299416320 00000100:00000001:0.0:1787829942.119343:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787829942.119344:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829942.119346:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787829942.119349:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829942/real 1787829942] req@ffff8d30c7017840 x1874670646880384/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829967 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787829942.119356:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787829942.119358:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787829942.119360:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787829942.119361:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787829942.119363:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829942.119364:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787829942.119365:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787829942.119369:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c7017840 x1874670646880384/t0(0) o38->lustre-MDT0001-mdc-ffff8d30c675d800@192.168.203.143@tcp:12/10 lens 520/544 e 0 to 1 dl 1787829967 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787829942.119381:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829942.119383:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787829942.119385:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829942.119386:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787829942.119387:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829942.119389:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787829942.119390:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787829942.119393:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f79000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829942.119395:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787829942.119397:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787829942.119400:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c7017840 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646880384:192.168.203.143@tcp:38:kworker.0 00000100:00000001:0.0:1787829942.119404:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829942.119405:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787829942.119406:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787829942.119408:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787829942.119410:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787829942.119411:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787829942.119412:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787829942.119414:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787829942.119415:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787829942.119417:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787829942.119418:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829942.119419:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787829942.119421:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787829942.119422:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787829942.119424:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787829942.119425:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787829942.119426:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787829942.119427:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787829942.119428:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787829942.119429:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787829942.119430:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829942.119432:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829942.119435:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787829942.119436:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829942.119438:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787829942.119439:0:2390:0:(client.c:1990:ptlrpc_check_set()) Process entered 00000100:00000001:0.0:1787829942.119441:0:2390:0:(client.c:1813:ptlrpc_send_new_req()) Process entered 00000100:00000001:0.0:1787829942.119442:0:2390:0:(client.c:1377:ptlrpc_import_delay_req()) Process entered 00000100:00000001:0.0:1787829942.119443:0:2390:0:(client.c:1436:ptlrpc_import_delay_req()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829942.119445:0:2390:0:(sec.c:709:sptlrpc_req_refresh_ctx()) Process entered 02000000:00000001:0.0:1787829942.119446:0:2390:0:(sec.c:740:sptlrpc_req_refresh_ctx()) Process leaving via out_sec_put (rc=0 : 0 : 0x0) 02000000:00000001:0.0:1787829942.119448:0:2390:0:(sec.c:784:sptlrpc_req_refresh_ctx()) Process leaving (rc=0 : 0 : 0) 00000100:00100000:0.0:1787829942.119451:0:2390:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d30c7014000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646880512:192.168.203.143@tcp:8:kworker.0 00000100:00000001:0.0:1787829942.119454:0:2390:0:(niobuf.c:866:ptl_send_rpc()) Process entered 02000000:00000001:0.0:1787829942.119455:0:2390:0:(sec.c:1080:sptlrpc_cli_wrap_request()) Process entered 02000000:00000001:0.0:1787829942.119457:0:2390:0:(sec.c:1117:sptlrpc_cli_wrap_request()) Process leaving (rc=0 : 0 : 0) 02000000:00000001:0.0:1787829942.119458:0:2390:0:(sec.c:1855:sptlrpc_cli_alloc_repbuf()) Process entered 02000000:00000001:0.0:1787829942.119461:0:2390:0:(sec.c:1865:sptlrpc_cli_alloc_repbuf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829942.119466:0:2390:0:(client.c:3275:ptlrpc_request_addref()) Process entered 00000100:00000001:0.0:1787829942.119467:0:2390:0:(client.c:3277:ptlrpc_request_addref()) Process leaving (rc=18446617839369535488 : -126234340016128 : ffff8d30c7014000) 00000100:00000001:0.0:1787829942.119471:0:2390:0:(niobuf.c:69:ptl_send_buf()) Process entered 00000800:00000100:0.0:1787829942.119481:0:2390:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.143@tcp 00000100:00000001:0.0:1787829942.119485:0:2390:0:(events.c:34:request_out_callback()) Process entered 00000100:00000001:0.0:1787829942.119487:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787829942.119489:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829942.119490:0:2390:0:(events.c:68:request_out_callback()) Process leaving 00000100:00000001:0.0:1787829942.119492:0:2390:0:(niobuf.c:111:ptl_send_buf()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829942.119495:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 0 is 4299415075 00000100:00000001:0.0:1787829942.119497:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 0, keep current deadline 4299416320 00000100:00000001:0.0:1787829942.119500:0:2390:0:(niobuf.c:663:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS new deadline estimation for cpu 1 is 4299415075 00000100:00000001:0.0:1787829942.119502:0:2390:0:(niobuf.c:692:kick_cpu_latency()) lustre-OST0000-osc-ffff8d30c675d800: PM QoS request active for cpu 1, keep current deadline 4299416320 00000100:00000001:0.0:1787829942.119503:0:2390:0:(niobuf.c:1135:ptl_send_rpc()) Process leaving via out (rc=0 : 0 : 0x0) 00000100:00000001:0.0:1787829942.119505:0:2390:0:(client.c:1941:ptlrpc_send_new_req()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829942.119507:0:2390:0:(client.c:2485:ptlrpc_expire_one_request()) Process entered 00000100:00100000:0.0:1787829942.119511:0:2390:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1787829942/real 1787829942] req@ffff8d30c7014000 x1874670646880512/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829967 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000400:00000001:0.0:1787829942.119523:0:2390:0:(lib-msg.c:22:lnet_build_unlink_event()) Process entered 00000400:00000001:0.0:1787829942.119525:0:2390:0:(lib-msg.c:31:lnet_build_unlink_event()) Process leaving 00000100:00000001:0.0:1787829942.119527:0:2390:0:(events.c:78:reply_in_callback()) Process entered 00000100:00000001:0.0:1787829942.119528:0:2390:0:(events.c:163:reply_in_callback()) Process leaving 00000100:00000001:0.0:1787829942.119530:0:2390:0:(client.c:3001:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829942.119531:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787829942.119533:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00100000:0.0:1787829942.119536:0:2390:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d30c7014000 x1874670646880512/t0(0) o8->lustre-OST0000-osc-ffff8d30c675d800@192.168.203.143@tcp:28/4 lens 520/544 e 0 to 1 dl 1787829967 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00000001:0.0:1787829942.119543:0:2390:0:(client.c:2539:ptlrpc_expire_one_request()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829942.119560:0:2390:0:(client.c:2124:ptlrpc_check_set()) Process leaving via interpret (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00000001:0.0:1787829942.119563:0:2390:0:(client.c:2993:ptlrpc_unregister_reply()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829942.119564:0:2390:0:(niobuf.c:473:ptlrpc_unregister_bulk()) Process entered 00000100:00000001:0.0:1787829942.119565:0:2390:0:(niobuf.c:487:ptlrpc_unregister_bulk()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829942.119567:0:2390:0:(import.c:1057:ptlrpc_connect_interpret()) Process entered 00000100:00000001:0.0:1787829942.119568:0:2390:0:(import.c:1092:ptlrpc_connect_interpret()) Process leaving via out (rc=18446744073709551506 : -110 : 0xffffffffffffff92) 00000100:00080000:0.0:1787829942.119571:0:2390:0:(import.c:65:import_set_state_nolock()) ffff8d30c9f7d000 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1787829942.119574:0:2390:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.143@tcp failed (-110) 00000100:00000001:0.0:1787829942.119576:0:2390:0:(import.c:1517:ptlrpc_connect_interpret()) Process leaving (rc=18446744073709551506 : -110 : ffffffffffffff92) 00000100:00100000:0.0:1787829942.119579:0:2390:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d30c7014000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:304cf071-b03b-42d4-83cb-76e070d32538:2390:1874670646880512:192.168.203.143@tcp:8:kworker.0 00000100:00000001:0.0:1787829942.119582:0:2390:0:(client.c:2465:ptlrpc_check_set()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829942.119584:0:2390:0:(client.c:2867:__ptlrpc_req_put()) Process entered 00000100:00000001:0.0:1787829942.119585:0:2390:0:(client.c:2793:__ptlrpc_free_req()) Process entered 02000000:00000001:0.0:1787829942.119586:0:2390:0:(sec.c:1880:sptlrpc_cli_free_repbuf()) Process entered 02000000:00000001:0.0:1787829942.119589:0:2390:0:(sec.c:1896:sptlrpc_cli_free_repbuf()) Process leaving 00000020:00000001:0.0:1787829942.119590:0:2390:0:(genops.c:1217:class_import_put()) Process entered 00000020:00000001:0.0:1787829942.119591:0:2390:0:(genops.c:1230:class_import_put()) Process leaving 02000000:00000001:0.0:1787829942.119593:0:2390:0:(sec.c:473:sptlrpc_req_put_ctx()) Process entered 02000000:00000001:0.0:1787829942.119594:0:2390:0:(sec.c:490:sptlrpc_req_put_ctx()) Process leaving 00000100:00000001:0.0:1787829942.119596:0:2390:0:(client.c:2849:__ptlrpc_free_req()) Process leaving 00000100:00000001:0.0:1787829942.119597:0:2390:0:(client.c:2908:__ptlrpc_req_put()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829942.119599:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=1 : 1 : 1) 00000100:00000001:0.0:1787829942.119636:0:2390:0:(client.c:2639:ptlrpc_set_next_timeout()) Process entered 00000100:00000001:0.0:1787829942.119638:0:2390:0:(client.c:2667:ptlrpc_set_next_timeout()) Process leaving (rc=0 : 0 : 0) 00000100:00000001:0.0:1787829942.119639:0:2390:0:(ptlrpcd.c:252:ptlrpcd_check()) Process entered 00000100:00000001:0.0:1787829942.119641:0:2390:0:(ptlrpcd.c:351:ptlrpcd_check()) Process leaving (rc=0 : 0 : 0) 00010000:00000001:2.0:1787829943.014709:0:815:0:(ldlm_pool.c:457:ldlm_cli_pool_recalc()) Process entered 00010000:00000001:2.0:1787829943.014713:0:815:0:(ldlm_pool.c:461:ldlm_cli_pool_recalc()) Process leaving (rc=0 : 0 : 0) 00000020:00000001:3.0:1787829943.014763:0:26958:0:(genops.c:1895:obd_stale_export_get()) Process entered 00000020:00000001:3.0:1787829943.014766:0:26958:0:(genops.c:1909:obd_stale_export_get()) Process leaving (rc=0 : 0 : 0) 00010000:00010000:3.0:1787829943.014768:0:26958:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) Debug log: 130914 lines, 130914 kept, 0 dropped, 0 bad.